From de0fdbc1ee7f233e9ae363c7162171135d3b7403 Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Sat, 29 Apr 2017 18:51:21 -0700
Subject: [PATCH 001/387] update JavaDoc for BoundedWindow

update JavaDoc

rephrase JavaDoc.
---
 .../sdk/transforms/windowing/BoundedWindow.java    | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
index 74223b586715..506753dc4f77 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
@@ -21,11 +21,15 @@
 import org.joda.time.Instant;
 
 /**
- * A {@code BoundedWindow} represents a finite grouping of elements, with an
- * upper bound (larger timestamps represent more recent data) on the timestamps
- * of elements that can be placed in the window. This finiteness means that for
- * every window, at some point in time, all data for that window will have
- * arrived and can be processed together.
+ * A {@code BoundedWindow} represents window information assigned to data elements.
+ *
+ * <p>It has one method {@link #maxTimestamp()} to define an upper bound(included) of element
+ * timestamp. Elements within the timestamp range are placed in this window.
+ * Technically a window may not have a lower bound. Upper bound is mandatory as watermark,
+ * which also controls discarding data and expiring the window, is based on event timestamp.
+ *
+ * <p>At some point in time, usually when watermark passes,
+ * all data with the same window will arrive and can be processed together.
  *
  * <p>Windows must also implement {@link Object#equals} and
  * {@link Object#hashCode} such that windows that are logically equal will

From f4dfbb206382d3ea73881727aa8b0f74eaf98ef4 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 19:31:22 -0700
Subject: [PATCH 002/387] Annotate internal methods of PCollection

---
 .../org/apache/beam/sdk/values/PCollection.java  | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
index 034f0de67340..20e5d68946b4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
@@ -22,6 +22,7 @@
 
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.CannotProvideCoderException.ReasonCode;
 import org.apache.beam.sdk.coders.Coder;
@@ -330,30 +331,27 @@ public PCollection<T> setTypeDescriptor(TypeDescriptor<T> typeDescriptor) {
   }
 
   /**
-   * Sets the {@link WindowingStrategy} of this {@link PCollection}.
-   *
-   * <p>For use by primitive transformations only.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
    */
+  @Internal
   public PCollection<T> setWindowingStrategyInternal(WindowingStrategy<?, ?> windowingStrategy) {
      this.windowingStrategy = windowingStrategy;
      return this;
   }
 
   /**
-   * Sets the {@link PCollection.IsBounded} of this {@link PCollection}.
-   *
-   * <p>For use by internal transformations only.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
    */
+  @Internal
   public PCollection<T> setIsBoundedInternal(IsBounded isBounded) {
     this.isBounded = isBounded;
     return this;
   }
 
   /**
-   * Creates and returns a new {@link PCollection} for a primitive output.
-   *
-   * <p>For use by primitive transformations only.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
    */
+  @Internal
   public static <T> PCollection<T> createPrimitiveOutputInternal(
       Pipeline pipeline,
       WindowingStrategy<?, ?> windowingStrategy,

From c1b26a1b53c334ab171fad60501ba67593fde5d2 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 19:48:38 -0700
Subject: [PATCH 003/387] Annotate internal pieces of sdks.transforms

---
 .../sdk/transforms/AppliedPTransform.java     |  7 ++--
 .../beam/sdk/transforms/CombineFnBase.java    | 22 ++++++++----
 .../beam/sdk/transforms/Materialization.java  |  6 +++-
 .../beam/sdk/transforms/Materializations.java | 11 ++++--
 .../org/apache/beam/sdk/transforms/View.java  | 36 ++++++++++++-------
 .../apache/beam/sdk/transforms/ViewFn.java    |  6 +++-
 6 files changed, 62 insertions(+), 26 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AppliedPTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AppliedPTransform.java
index bdb61b834026..4e049a590d7e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AppliedPTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AppliedPTransform.java
@@ -20,16 +20,14 @@
 import com.google.auto.value.AutoValue;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 
 /**
- * Represents the application of a {@link PTransform} to a specific input to produce
- * a specific output.
- *
- * <p>For internal use.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  *
  * <p>Inputs and outputs are stored in their expanded forms, as the condensed form of a composite
  * {@link PInput} or {@link POutput} is a language-specific concept, and {@link AppliedPTransform}
@@ -40,6 +38,7 @@
  * @param <OutputT>    transform output type
  * @param <TransformT> transform type
  */
+@Internal
 @AutoValue
 public abstract class AppliedPTransform<
     InputT extends PInput, OutputT extends POutput,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
index a88109968e35..29990cd60b00 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
@@ -21,6 +21,7 @@
 import java.io.Serializable;
 import java.lang.reflect.Type;
 import java.lang.reflect.TypeVariable;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
@@ -31,14 +32,19 @@
 import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
- * This class contains the shared interfaces and abstract classes for different types of combine
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>This class contains the shared interfaces and abstract classes for different types of combine
  * functions.
  *
  * <p>Users should not implement or extend them directly.
  */
+@Internal
 public class CombineFnBase {
   /**
-   * A {@code GloballyCombineFn<InputT, AccumT, OutputT>} specifies how to combine a
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>A {@code GloballyCombineFn<InputT, AccumT, OutputT>} specifies how to combine a
    * collection of input values of type {@code InputT} into a single
    * output value of type {@code OutputT}.  It does this via one or more
    * intermediate mutable accumulator values of type {@code AccumT}.
@@ -50,6 +56,7 @@ public class CombineFnBase {
    * @param <AccumT> type of mutable accumulator values
    * @param <OutputT> type of output values
    */
+  @Internal
   public interface GlobalCombineFn<InputT, AccumT, OutputT> extends Serializable, HasDisplayData {
 
     /**
@@ -93,16 +100,19 @@ Coder<OutputT> getDefaultOutputCoder(CoderRegistry registry, Coder<InputT> input
   }
 
   /**
-   * An abstract {@link GlobalCombineFn} base class shared by
-   * {@link CombineFn} and {@link CombineFnWithContext}.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
    *
-   * <p>Do not extend this class directly.
-   * Extends {@link CombineFn} and {@link CombineFnWithContext} instead.
+   * <p>An abstract {@link GlobalCombineFn} base class shared by {@link CombineFn} and {@link
+   * CombineFnWithContext}.
+   *
+   * <p>Do not extend this class directly. Extends {@link CombineFn} and {@link
+   * CombineFnWithContext} instead.
    *
    * @param <InputT> type of input values
    * @param <AccumT> type of mutable accumulator values
    * @param <OutputT> type of output values
    */
+  @Internal
   abstract static class AbstractGlobalCombineFn<InputT, AccumT, OutputT>
       implements GlobalCombineFn<InputT, AccumT, OutputT>, Serializable {
     private static final String INCOMPATIBLE_GLOBAL_WINDOW_ERROR_MESSAGE =
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java
index 7cd6256d9f00..6fb8c297f547 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java
@@ -18,15 +18,19 @@
 
 package org.apache.beam.sdk.transforms;
 
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.runners.PipelineRunner;
 
 /**
- * How a view should be physically materialized by a {@link PipelineRunner}.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>How a view should be physically materialized by a {@link PipelineRunner}.
  *
  * <p>A {@link PipelineRunner} will support some set of materializations, and will reject
  * {@link ViewFn ViewFns} that require materializations it does not support. See
  * {@link Materializations} for known implementations.
  */
+@Internal
 public interface Materialization<T> {
   /**
    * Gets the URN describing this {@link Materialization}. This is a stable, SDK-independent URN
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java
index 35925fae5f7a..6e4f83d6f6a1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java
@@ -20,12 +20,16 @@
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.util.WindowedValue;
 
 /**
- * Utility methods for constructing known {@link Materialization materializations} for a
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>Utility methods for constructing known {@link Materialization materializations} for a
  * {@link ViewFn}.
  */
+@Internal
 public class Materializations {
   /**
    * The URN for a {@link Materialization} where the primitive view type is an iterable of fully
@@ -36,9 +40,12 @@ public class Materializations {
       "urn:beam:sideinput:materialization:iterable:0.1";
 
   /**
-   * A {@link Materialization} where the primitive view type is an iterable of fully specified
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>A {@link Materialization} where the primitive view type is an iterable of fully specified
    * windowed values.
    */
+  @Internal
   public static <T> Materialization<Iterable<WindowedValue<T>>> iterable() {
     return new IterableMaterialization<>();
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
index b3b8918d411b..d17d423441a9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
@@ -19,6 +19,7 @@
 
 import java.util.List;
 import java.util.Map;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.runners.PipelineRunner;
@@ -237,11 +238,13 @@ public static <K, V> AsMultimap<K, V> asMultimap() {
   }
 
   /**
-   * Not intended for direct use by pipeline authors; public only so a {@link PipelineRunner} may
-   * override its behavior.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Public only so a {@link PipelineRunner} may override its behavior.
    *
    * <p>See {@link View#asList()}.
    */
+  @Internal
   public static class AsList<T> extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
     private AsList() { }
 
@@ -259,11 +262,13 @@ public PCollectionView<List<T>> expand(PCollection<T> input) {
   }
 
   /**
-   * Not intended for direct use by pipeline authors; public only so a {@link PipelineRunner} may
-   * override its behavior.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Public only so a {@link PipelineRunner} may override its behavior.
    *
    * <p>See {@link View#asIterable()}.
    */
+  @Internal
   public static class AsIterable<T>
       extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
     private AsIterable() { }
@@ -282,11 +287,13 @@ public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
   }
 
   /**
-   * Not intended for direct use by pipeline authors; public only so a {@link PipelineRunner} may
-   * override its behavior.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Public only so a {@link PipelineRunner} may override its behavior.
    *
    * <p>See {@link View#asSingleton()}.
    */
+  @Internal
   public static class AsSingleton<T> extends PTransform<PCollection<T>, PCollectionView<T>> {
     private final T defaultValue;
     private final boolean hasDefault;
@@ -396,11 +403,13 @@ public T identity() {
   }
 
   /**
-   * Not intended for direct use by pipeline authors; public only so a {@link PipelineRunner} may
-   * override its behavior.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Public only so a {@link PipelineRunner} may override its behavior.
    *
    * <p>See {@link View#asMultimap()}.
    */
+  @Internal
   public static class AsMultimap<K, V>
       extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, Iterable<V>>>> {
     private AsMultimap() { }
@@ -422,11 +431,13 @@ public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input)
   }
 
   /**
-   * Not intended for direct use by pipeline authors; public only so a {@link PipelineRunner} may
-   * override its behavior.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Public only so a {@link PipelineRunner} may override its behavior.
    *
    * <p>See {@link View#asMap()}.
    */
+  @Internal
   public static class AsMap<K, V>
       extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
     private AsMap() { }
@@ -459,13 +470,14 @@ public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
   // Internal details below
 
   /**
-   * Creates a primitive {@link PCollectionView}.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
    *
-   * <p>For internal use only by runner implementors.
+   * <p>Creates a primitive {@link PCollectionView}.
    *
    * @param <ElemT> The type of the elements of the input PCollection
    * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
    */
+  @Internal
   public static class CreatePCollectionView<ElemT, ViewT>
       extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
     private PCollectionView<ViewT> view;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java
index cdfcb88652dc..d51a9171035f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java
@@ -18,11 +18,14 @@
 package org.apache.beam.sdk.transforms;
 
 import java.io.Serializable;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
- * A function to adapt a primitive "view" of a {@link PCollection} - some materialization
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>A function to adapt a primitive "view" of a {@link PCollection} - some materialization
  * specified in the Beam model and implemented by the runner - to a user-facing view type
  * for side input.
  *
@@ -36,6 +39,7 @@
  * @param <PrimitiveViewT> the type of the underlying primitive view, provided by the runner
  *        {@code <ViewT>} the type of the value(s) accessible via this {@link PCollectionView}
  */
+@Internal
 public abstract class ViewFn<PrimitiveViewT, ViewT> implements Serializable {
   /**
    * Gets the materialization of this {@link ViewFn}.

From 49cf433c5c08f3cc91512aa9544a36a5d3e84333 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 19:59:32 -0700
Subject: [PATCH 004/387] Tighten access control and internal annotations for
 triggers

---
 .../sdk/transforms/windowing/AfterAll.java    |  4 +-
 .../sdk/transforms/windowing/AfterEach.java   |  2 +-
 .../sdk/transforms/windowing/AfterFirst.java  |  2 +-
 .../sdk/transforms/windowing/AfterPane.java   |  2 +-
 .../windowing/AfterProcessingTime.java        |  2 +-
 .../transforms/windowing/AfterWatermark.java  |  4 +-
 .../transforms/windowing/DefaultTrigger.java  |  2 +-
 .../beam/sdk/transforms/windowing/Never.java  |  2 +-
 .../windowing/OrFinallyTrigger.java           |  2 +-
 .../sdk/transforms/windowing/Repeatedly.java  |  2 +-
 .../windowing/TimestampTransform.java         | 41 +++++++++++++++----
 .../sdk/transforms/windowing/Trigger.java     | 18 ++++++--
 12 files changed, 62 insertions(+), 21 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java
index 2747311dff98..eb0a7acbcac6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java
@@ -23,6 +23,7 @@
 import java.util.Arrays;
 import java.util.List;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger;
 import org.joda.time.Instant;
 
@@ -51,6 +52,7 @@ public static AfterAll of(List<Trigger> triggers) {
     return new AfterAll(triggers);
   }
 
+  @Internal
   @Override
   public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
     // This trigger will fire after the latest of its sub-triggers.
@@ -65,7 +67,7 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
   }
 
   @Override
-  public OnceTrigger getContinuationTrigger(List<Trigger> continuationTriggers) {
+  protected OnceTrigger getContinuationTrigger(List<Trigger> continuationTriggers) {
     return new AfterAll(continuationTriggers);
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java
index 56a9d14a2487..1fc4fbf52a97 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java
@@ -72,7 +72,7 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
   }
 
   @Override
-  public Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
+  protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
     return Repeatedly.forever(new AfterFirst(continuationTriggers));
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java
index 79fd6391d60a..f0beb0a6d82b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java
@@ -66,7 +66,7 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
   }
 
   @Override
-  public OnceTrigger getContinuationTrigger(List<Trigger> continuationTriggers) {
+  protected OnceTrigger getContinuationTrigger(List<Trigger> continuationTriggers) {
     return new AfterFirst(continuationTriggers);
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java
index 25c55936a223..eade95d1350a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java
@@ -61,7 +61,7 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
   }
 
   @Override
-  public OnceTrigger getContinuationTrigger(List<Trigger> continuationTriggers) {
+  protected OnceTrigger getContinuationTrigger(List<Trigger> continuationTriggers) {
     return AfterPane.elementCountAtLeast(1);
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java
index eda269adf248..cc7ec137b87f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java
@@ -40,7 +40,7 @@ public class AfterProcessingTime extends OnceTrigger {
 
   private final List<TimestampTransform> timestampTransforms;
 
-  public AfterProcessingTime(List<TimestampTransform> timestampTransforms) {
+  private AfterProcessingTime(List<TimestampTransform> timestampTransforms) {
     super(null);
     this.timestampTransforms = timestampTransforms;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java
index 6825ab08d1fc..14a8c98a7787 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java
@@ -88,7 +88,7 @@ public OnceTrigger getLateTrigger() {
     }
 
     @SuppressWarnings("unchecked")
-    public AfterWatermarkEarlyAndLate(OnceTrigger earlyTrigger, OnceTrigger lateTrigger) {
+    private AfterWatermarkEarlyAndLate(OnceTrigger earlyTrigger, OnceTrigger lateTrigger) {
       super(lateTrigger == null
           ? ImmutableList.<Trigger>of(earlyTrigger)
           : ImmutableList.<Trigger>of(earlyTrigger, lateTrigger));
@@ -178,7 +178,7 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
     }
 
     @Override
-    public FromEndOfWindow getContinuationTrigger(List<Trigger> continuationTriggers) {
+    protected FromEndOfWindow getContinuationTrigger(List<Trigger> continuationTriggers) {
       return this;
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java
index a649b4ff1f51..78f373540fd2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java
@@ -51,7 +51,7 @@ public boolean isCompatible(Trigger other) {
   }
 
   @Override
-  public Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
+  protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
     return this;
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java
index 664ae83460e8..6dfeea72067f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java
@@ -43,7 +43,7 @@ public static NeverTrigger ever() {
    * The actual trigger class for {@link Never} triggers.
    */
   public static class NeverTrigger extends OnceTrigger {
-    protected NeverTrigger() {
+    private NeverTrigger() {
       super(null);
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java
index 1ed9b550ca99..ad0de47e0bbb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java
@@ -58,7 +58,7 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
   }
 
   @Override
-  public Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
+  protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
     // Use OrFinallyTrigger instead of AfterFirst because the continuation of ACTUAL
     // may not be a OnceTrigger.
     return Repeatedly.forever(
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java
index 40591e3f8bd4..78b79c74fbab 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java
@@ -66,7 +66,7 @@ public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
   }
 
   @Override
-  public Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
+  protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) {
     return new Repeatedly(continuationTriggers.get(REPEATED));
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java
index 5318592f5141..8bdf6ee0d486 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java
@@ -19,37 +19,59 @@
 
 import com.google.auto.value.AutoValue;
 import java.io.Serializable;
+import org.apache.beam.sdk.annotations.Internal;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
-/** An abstract description of a standardized transformation on timestamps. */
+/**
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>An abstract description of a standardized transformation on timestamps.
+ */
+@Internal
 public abstract class TimestampTransform implements Serializable{
 
-  /** Returns a transform that shifts a timestamp later by {@code delay}. */
+  TimestampTransform() {}
+
+  /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Returns a transform that shifts a timestamp later by {@code delay}.
+   */
+  @Internal
   public static TimestampTransform delay(Duration delay) {
     return new AutoValue_TimestampTransform_Delay(delay);
   }
 
   /**
-   * Returns a transform that aligns a timestamp to the next boundary of {@code period}, starting
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Returns a transform that aligns a timestamp to the next boundary of {@code period}, starting
    * from {@code offset}.
    */
+  @Internal
   public static TimestampTransform alignTo(Duration period, Instant offset) {
     return new AutoValue_TimestampTransform_AlignTo(period, offset);
   }
 
   /**
-   * Returns a transform that aligns a timestamp to the next boundary of {@code period}, starting
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Returns a transform that aligns a timestamp to the next boundary of {@code period}, starting
    * from the start of the epoch.
    */
+  @Internal
   public static TimestampTransform alignTo(Duration period) {
     return alignTo(period, new Instant(0));
   }
 
   /**
-   * Represents the transform that aligns a timestamp to the next boundary of {@link #getPeriod()}
-   * start at {@link #getOffset()}.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Represents the transform that aligns a timestamp to the next boundary of {@link
+   * #getPeriod()} start at {@link #getOffset()}.
    */
+  @Internal
   @AutoValue
   public abstract static class AlignTo extends TimestampTransform {
     public abstract Duration getPeriod();
@@ -57,7 +79,12 @@ public abstract static class AlignTo extends TimestampTransform {
     public abstract Instant getOffset();
   }
 
-  /** Represents the transform that delays a timestamp by {@link #getDelay()}. */
+  /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Represents the transform that delays a timestamp by {@link #getDelay()}.
+   */
+  @Internal
   @AutoValue
   public abstract static class Delay extends TimestampTransform {
     public abstract Duration getDelay();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java
index 07d30779b079..519ab6789857 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Objects;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.joda.time.Instant;
 
@@ -117,8 +118,10 @@ public Trigger getContinuationTrigger() {
   protected abstract Trigger getContinuationTrigger(List<Trigger> continuationTriggers);
 
   /**
-   * Returns a bound in event time by which this trigger would have fired at least once for a given
-   * window had there been input data.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Returns a bound in event time by which this trigger would have fired at least once for a
+   * given window had there been input data.
    *
    * <p>For triggers that do not fire based on the watermark advancing, returns {@link
    * BoundedWindow#TIMESTAMP_MAX_VALUE}.
@@ -126,9 +129,15 @@ public Trigger getContinuationTrigger() {
    * <p>This estimate may be used, for example, to determine that there are no elements in a
    * side-input window, which causes the default value to be used instead.
    */
+  @Internal
   public abstract Instant getWatermarkThatGuaranteesFiring(BoundedWindow window);
 
-  /** Returns whether this performs the same triggering as the given {@link Trigger}. */
+  /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Returns whether this performs the same triggering as the given {@link Trigger}.
+   */
+  @Internal
   public boolean isCompatible(Trigger other) {
     if (!getClass().equals(other.getClass())) {
       return false;
@@ -208,9 +217,12 @@ public OrFinallyTrigger orFinally(OnceTrigger until) {
   }
 
   /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
    * {@link Trigger Triggers} that are guaranteed to fire at most once should extend {@link
    * OnceTrigger} rather than the general {@link Trigger} class to indicate that behavior.
    */
+  @Internal
   public abstract static class OnceTrigger extends Trigger {
     protected OnceTrigger(List<Trigger> subTriggers) {
       super(subTriggers);

From 9b8a4e5c4b876d4459c64a9bffee613aeae72fb2 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 20:05:34 -0700
Subject: [PATCH 005/387] The transforms.reflect package is not for users

---
 .../org/apache/beam/sdk/transforms/reflect/package-info.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java
index 4df5209e6b39..fe2f6b1f8cad 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/package-info.java
@@ -16,8 +16,11 @@
  * limitations under the License.
  */
 /**
- * Defines reflection-based utilities for analyzing {@link org.apache.beam.sdk.transforms.DoFn}'s
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>Defines reflection-based utilities for analyzing {@link org.apache.beam.sdk.transforms.DoFn}'s
  * and creating {@link org.apache.beam.sdk.transforms.reflect.DoFnSignature}'s and
  * {@link org.apache.beam.sdk.transforms.reflect.DoFnInvoker}'s from them.
  */
 package org.apache.beam.sdk.transforms.reflect;
+

From fe51cc0d1a8aa14adbee81b220f9ca8a442f26fe Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 20:05:45 -0700
Subject: [PATCH 006/387] Annotate internal-only bits of Java sdk.runners

---
 .../java/org/apache/beam/sdk/runners/PTransformMatcher.java | 6 +++++-
 .../org/apache/beam/sdk/runners/PTransformOverride.java     | 6 +++++-
 .../apache/beam/sdk/runners/PTransformOverrideFactory.java  | 6 +++++-
 .../apache/beam/sdk/runners/PipelineRunnerRegistrar.java    | 4 ++++
 .../org/apache/beam/sdk/runners/TransformHierarchy.java     | 6 +++++-
 5 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java
index 30dca6d1dcee..6378ecc82f6e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java
@@ -20,12 +20,16 @@
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
- * Matches applications of {@link PTransform PTransforms}.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>Matches applications of {@link PTransform PTransforms}.
  */
+@Internal
 @Experimental(Kind.CORE_RUNNERS_ONLY)
 public interface PTransformMatcher {
   boolean matches(AppliedPTransform<?, ?, ?> application);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverride.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverride.java
index 33b9114abd57..2820364d82eb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverride.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverride.java
@@ -19,12 +19,16 @@
 package org.apache.beam.sdk.runners;
 
 import com.google.auto.value.AutoValue;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
- * A {@link PTransformMatcher} and associated {@link PTransformOverrideFactory} to replace all
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>A {@link PTransformMatcher} and associated {@link PTransformOverrideFactory} to replace all
  * matching {@link PTransform PTransforms}.
  */
+@Internal
 @AutoValue
 public abstract class PTransformOverride {
   public static PTransformOverride of(
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
index 786c61c95aa1..a28f303688d6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PInput;
@@ -32,9 +33,12 @@
 import org.apache.beam.sdk.values.TupleTag;
 
 /**
- * Produces {@link PipelineRunner}-specific overrides of {@link PTransform PTransforms}, and
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>Produces {@link PipelineRunner}-specific overrides of {@link PTransform PTransforms}, and
  * provides mappings between original and replacement outputs.
  */
+@Internal
 @Experimental(Kind.CORE_RUNNERS_ONLY)
 public interface PTransformOverrideFactory<
     InputT extends PInput,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java
index be95044bb6c5..41fd6f0029ba 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java
@@ -19,8 +19,11 @@
 
 import com.google.auto.service.AutoService;
 import java.util.ServiceLoader;
+import org.apache.beam.sdk.annotations.Internal;
 
 /**
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
  * {@link PipelineRunner} creators have the ability to automatically have their
  * {@link PipelineRunner} registered with this SDK by creating a {@link ServiceLoader} entry
  * and a concrete implementation of this interface.
@@ -33,6 +36,7 @@
  * <p>It is optional but recommended to use one of the many build time tools such as
  * {@link AutoService} to generate the necessary META-INF files automatically.
  */
+@Internal
 public interface PipelineRunnerRegistrar {
   /**
    * Get the set of {@link PipelineRunner PipelineRunners} to register.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 18bf2e95e8b9..92361942fc8a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -36,6 +36,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -47,9 +48,12 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Captures information about a collection of transformations and their
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>Captures information about a collection of transformations and their
  * associated {@link PValue}s.
  */
+@Internal
 public class TransformHierarchy {
   private static final Logger LOG = LoggerFactory.getLogger(TransformHierarchy.class);
 

From 58298d866fe9d1f4fcaf2ccda3078809f4d55b27 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 10:10:07 -0700
Subject: [PATCH 007/387] Tighten access in sdk.options

---
 .../{ValueProviderUtils.java => ValueProviders.java}   |  4 ++--
 .../org/apache/beam/sdk/options/ValueProviderTest.java |  4 ++--
 ...eProviderUtilsTest.java => ValueProvidersTest.java} | 10 +++++-----
 3 files changed, 9 insertions(+), 9 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/options/{ValueProviderUtils.java => ValueProviders.java} (96%)
 rename sdks/java/core/src/test/java/org/apache/beam/sdk/options/{ValueProviderUtilsTest.java => ValueProvidersTest.java} (90%)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviderUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
similarity index 96%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviderUtils.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
index 14a5f2391a66..d034b819f40f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviderUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
@@ -28,8 +28,8 @@
 /**
  * Utilities for working with the {@link ValueProvider} interface.
  */
-public class ValueProviderUtils {
-  private ValueProviderUtils() {}
+class ValueProviders {
+  private ValueProviders() {}
 
   /**
    * Given {@code serializedOptions} as a JSON-serialized {@link PipelineOptions}, updates
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
index 383de535299d..9369ae6957c1 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
@@ -199,7 +199,7 @@ public void testSerializeDeserializeNoArg() throws Exception {
     ObjectMapper mapper = new ObjectMapper();
     String serializedOptions = mapper.writeValueAsString(submitOptions);
 
-    String runnerString = ValueProviderUtils.updateSerializedOptions(
+    String runnerString = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("foo", "quux"));
     TestOptions runtime = mapper.readValue(runnerString, PipelineOptions.class)
       .as(TestOptions.class);
@@ -218,7 +218,7 @@ public void testSerializeDeserializeWithArg() throws Exception {
     ObjectMapper mapper = new ObjectMapper();
     String serializedOptions = mapper.writeValueAsString(submitOptions);
 
-    String runnerString = ValueProviderUtils.updateSerializedOptions(
+    String runnerString = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("foo", "quux"));
     TestOptions runtime = mapper.readValue(runnerString, PipelineOptions.class)
       .as(TestOptions.class);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java
similarity index 90%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderUtilsTest.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java
index e09f4ad3af2e..14f86bc279e0 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderUtilsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java
@@ -26,9 +26,9 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
-/** Tests for {@link ValueProviderUtils}. */
+/** Tests for {@link ValueProviders}. */
 @RunWith(JUnit4.class)
-public class ValueProviderUtilsTest {
+public class ValueProvidersTest {
   /** A test interface. */
   public interface TestOptions extends PipelineOptions {
     String getString();
@@ -43,7 +43,7 @@ public void testUpdateSerialize() throws Exception {
     TestOptions submitOptions = PipelineOptionsFactory.as(TestOptions.class);
     ObjectMapper mapper = new ObjectMapper();
     String serializedOptions = mapper.writeValueAsString(submitOptions);
-    String updatedOptions = ValueProviderUtils.updateSerializedOptions(
+    String updatedOptions = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("string", "bar"));
     TestOptions runtime = mapper.readValue(updatedOptions, PipelineOptions.class)
       .as(TestOptions.class);
@@ -56,7 +56,7 @@ public void testUpdateSerializeExistingValue() throws Exception {
         "--string=baz", "--otherString=quux").as(TestOptions.class);
     ObjectMapper mapper = new ObjectMapper();
     String serializedOptions = mapper.writeValueAsString(submitOptions);
-    String updatedOptions = ValueProviderUtils.updateSerializedOptions(
+    String updatedOptions = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("string", "bar"));
     TestOptions runtime = mapper.readValue(updatedOptions, PipelineOptions.class)
       .as(TestOptions.class);
@@ -69,7 +69,7 @@ public void testUpdateSerializeEmptyUpdate() throws Exception {
     TestOptions submitOptions = PipelineOptionsFactory.as(TestOptions.class);
     ObjectMapper mapper = new ObjectMapper();
     String serializedOptions = mapper.writeValueAsString(submitOptions);
-    String updatedOptions = ValueProviderUtils.updateSerializedOptions(
+    String updatedOptions = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.<String, String>of());
     TestOptions runtime = mapper.readValue(updatedOptions, PipelineOptions.class)
       .as(TestOptions.class);

From 362d0be79222ad67f1639d54434c1505ef76752b Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 10:13:15 -0700
Subject: [PATCH 008/387] Annotate internal methods on Pipeline

---
 .../java/org/apache/beam/sdk/Pipeline.java    | 29 ++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index 351e1b8170dd..6b15f0d3f655 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -28,6 +28,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -190,12 +191,15 @@ public <OutputT extends POutput> OutputT apply(
   }
 
   /**
-   * Replaces all nodes that match a {@link PTransformOverride} in this pipeline. Overrides are
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Replaces all nodes that match a {@link PTransformOverride} in this pipeline. Overrides are
    * applied in the order they are present within the list.
    *
    * <p>After all nodes are replaced, ensures that no nodes in the updated graph match any of the
    * overrides.
    */
+  @Internal
   public void replaceAll(List<PTransformOverride> overrides) {
     for (PTransformOverride override : overrides) {
       replace(override);
@@ -334,10 +338,12 @@ public void setCoderRegistry(CoderRegistry coderRegistry) {
   }
 
   /**
-   * A {@link PipelineVisitor} can be passed into
-   * {@link Pipeline#traverseTopologically} to be called for each of the
-   * transforms and values in the {@link Pipeline}.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>A {@link PipelineVisitor} can be passed into {@link Pipeline#traverseTopologically} to be
+   * called for each of the transforms and values in the {@link Pipeline}.
    */
+  @Internal
   public interface PipelineVisitor {
     /**
      * Called for each composite transform after all topological predecessors have been visited
@@ -396,7 +402,9 @@ public void visitValue(PValue value, TransformHierarchy.Node producer) { }
   }
 
   /**
-   * Invokes the {@link PipelineVisitor PipelineVisitor's}
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Invokes the {@link PipelineVisitor PipelineVisitor's}
    * {@link PipelineVisitor#visitPrimitiveTransform} and
    * {@link PipelineVisitor#visitValue} operations on each of this
    * {@link Pipeline Pipeline's} transform and value nodes, in forward
@@ -408,14 +416,18 @@ public void visitValue(PValue value, TransformHierarchy.Node producer) { }
    *
    * <p>Typically invoked by {@link PipelineRunner} subclasses.
    */
+  @Internal
   public void traverseTopologically(PipelineVisitor visitor) {
     transforms.visit(visitor);
   }
 
   /**
-   * Like {@link #applyTransform(String, PInput, PTransform)} but defaulting to the name
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Like {@link #applyTransform(String, PInput, PTransform)} but defaulting to the name
    * provided by the {@link PTransform}.
    */
+  @Internal
   public static <InputT extends PInput, OutputT extends POutput>
   OutputT applyTransform(InputT input,
       PTransform<? super InputT, OutputT> transform) {
@@ -423,7 +435,9 @@ OutputT applyTransform(InputT input,
   }
 
   /**
-   * Applies the given {@code PTransform} to this input {@code InputT} and returns
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Applies the given {@code PTransform} to this input {@code InputT} and returns
    * its {@code OutputT}. This uses {@code name} to identify this specific application
    * of the transform. This name is used in various places, including the monitoring UI,
    * logging, and to stably identify this application node in the {@link Pipeline} graph during
@@ -432,6 +446,7 @@ OutputT applyTransform(InputT input,
    * <p>Each {@link PInput} subclass that provides an {@code apply} method should delegate to
    * this method to ensure proper registration with the {@link PipelineRunner}.
    */
+  @Internal
   public static <InputT extends PInput, OutputT extends POutput>
   OutputT applyTransform(String name, InputT input,
       PTransform<? super InputT, OutputT> transform) {

From 1f1c897264ea7ab050c8644344f6e2648af9ae4a Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Wed, 3 May 2017 17:17:11 -0700
Subject: [PATCH 009/387] [BEAM-2165] Update Apex to support
 serializing/deserializing custom user types configured via Jackson modules

---
 runners/apex/pom.xml                          |  4 +
 .../utils/SerializablePipelineOptions.java    | 15 ++-
 .../utils/PipelineOptionsTest.java            | 98 ++++++++++++++++---
 3 files changed, 99 insertions(+), 18 deletions(-)

diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index 36252e8c0530..aa4bddf57f76 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -62,6 +62,10 @@
       <artifactId>malhar-library</artifactId>
       <version>${apex.malhar.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
     <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
index 1a47ed574e2b..14476b57c26c 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.runners.apex.translation.utils;
 
+import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.Externalizable;
 import java.io.IOException;
@@ -27,6 +28,7 @@
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 
 /**
  * A wrapper to enable serialization of {@link PipelineOptions}.
@@ -51,13 +53,13 @@ public ApexPipelineOptions get() {
 
   @Override
   public void writeExternal(ObjectOutput out) throws IOException {
-    out.writeUTF(new ObjectMapper().writeValueAsString(pipelineOptions));
+    out.writeUTF(createMapper().writeValueAsString(pipelineOptions));
   }
 
   @Override
   public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
     String s = in.readUTF();
-    this.pipelineOptions = new ObjectMapper().readValue(s, PipelineOptions.class)
+    this.pipelineOptions = createMapper().readValue(s, PipelineOptions.class)
         .as(ApexPipelineOptions.class);
 
     if (FILE_SYSTEMS_INTIIALIZED.compareAndSet(false, true)) {
@@ -66,4 +68,13 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept
     }
   }
 
+  /**
+   * Use an {@link ObjectMapper} configured with any {@link Module}s in the class path allowing
+   * for user specified configuration injection into the ObjectMapper. This supports user custom
+   * types on {@link PipelineOptions}.
+   */
+  private static ObjectMapper createMapper() {
+    return new ObjectMapper().registerModules(
+        ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+  }
 }
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java
index d5eb9a98d645..118ff996bfa8 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/PipelineOptionsTest.java
@@ -23,15 +23,25 @@
 import com.datatorrent.common.util.FSStorageAgent;
 import com.esotericsoftware.kryo.serializers.FieldSerializer.Bind;
 import com.esotericsoftware.kryo.serializers.JavaSerializer;
-
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.auto.service.AutoService;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-
+import java.io.IOException;
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 /**
@@ -49,36 +59,92 @@ public interface MyOptions extends ApexPipelineOptions {
     void setTestOption(String value);
   }
 
-  private static class MyOptionsWrapper {
-    private MyOptionsWrapper() {
+  private static class OptionsWrapper {
+    private OptionsWrapper() {
       this(null); // required for Kryo
     }
-    private MyOptionsWrapper(ApexPipelineOptions options) {
+    private OptionsWrapper(ApexPipelineOptions options) {
       this.options = new SerializablePipelineOptions(options);
     }
     @Bind(JavaSerializer.class)
     private final SerializablePipelineOptions options;
   }
 
-  private static MyOptions options;
-
-  private static final String[] args = new String[]{"--testOption=nothing"};
+  @Test
+  public void testSerialization() {
+    OptionsWrapper wrapper = new OptionsWrapper(
+        PipelineOptionsFactory.fromArgs("--testOption=nothing").as(MyOptions.class));
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    FSStorageAgent.store(bos, wrapper);
 
-  @BeforeClass
-  public static void beforeTest() {
-    options = PipelineOptionsFactory.fromArgs(args).as(MyOptions.class);
+    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+    OptionsWrapper wrapperCopy = (OptionsWrapper) FSStorageAgent.retrieve(bis);
+    assertNotNull(wrapperCopy.options);
+    assertEquals("nothing", wrapperCopy.options.get().as(MyOptions.class).getTestOption());
   }
 
   @Test
-  public void testSerialization() {
-    MyOptionsWrapper wrapper = new MyOptionsWrapper(PipelineOptionsTest.options);
+  public void testSerializationWithUserCustomType() {
+    OptionsWrapper wrapper = new OptionsWrapper(
+        PipelineOptionsFactory.fromArgs("--jacksonIncompatible=\"testValue\"")
+            .as(JacksonIncompatibleOptions.class));
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     FSStorageAgent.store(bos, wrapper);
 
     ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
-    MyOptionsWrapper wrapperCopy = (MyOptionsWrapper) FSStorageAgent.retrieve(bis);
+    OptionsWrapper wrapperCopy = (OptionsWrapper) FSStorageAgent.retrieve(bis);
     assertNotNull(wrapperCopy.options);
-    assertEquals("nothing", wrapperCopy.options.get().as(MyOptions.class).getTestOption());
+    assertEquals("testValue",
+        wrapperCopy.options.get().as(JacksonIncompatibleOptions.class)
+            .getJacksonIncompatible().value);
+  }
+
+  /** PipelineOptions used to test auto registration of Jackson modules. */
+  public interface JacksonIncompatibleOptions extends ApexPipelineOptions {
+    JacksonIncompatible getJacksonIncompatible();
+    void setJacksonIncompatible(JacksonIncompatible value);
   }
 
+  /** A Jackson {@link Module} to test auto-registration of modules. */
+  @AutoService(Module.class)
+  public static class RegisteredTestModule extends SimpleModule {
+    public RegisteredTestModule() {
+      super("RegisteredTestModule");
+      setMixInAnnotation(JacksonIncompatible.class, JacksonIncompatibleMixin.class);
+    }
+  }
+
+  /** A class which Jackson does not know how to serialize/deserialize. */
+  public static class JacksonIncompatible {
+    private final String value;
+    public JacksonIncompatible(String value) {
+      this.value = value;
+    }
+  }
+
+  /** A Jackson mixin used to add annotations to other classes. */
+  @JsonDeserialize(using = JacksonIncompatibleDeserializer.class)
+  @JsonSerialize(using = JacksonIncompatibleSerializer.class)
+  public static final class JacksonIncompatibleMixin {}
+
+  /** A Jackson deserializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleDeserializer extends
+      JsonDeserializer<JacksonIncompatible> {
+
+    @Override
+    public JacksonIncompatible deserialize(JsonParser jsonParser,
+        DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+      return new JacksonIncompatible(jsonParser.readValueAs(String.class));
+    }
+  }
+
+  /** A Jackson serializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleSerializer extends JsonSerializer<JacksonIncompatible> {
+
+    @Override
+    public void serialize(JacksonIncompatible jacksonIncompatible, JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
+      jsonGenerator.writeString(jacksonIncompatible.value);
+    }
+  }
 }

From e5729b58330a05e7be510710d0027c004704946b Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Wed, 3 May 2017 17:19:00 -0700
Subject: [PATCH 010/387] [BEAM-2165] Update Dataflow to support
 serializing/deserializing custom user types configured via Jackson modules

This also updates the runner harness and existing tests to use a properly constructed ObjectMapper for PipelineOptions.
---
 .../dataflow/DataflowPipelineTranslator.java  | 13 ++-
 .../DataflowPipelineTranslatorTest.java       | 83 +++++++++++++++++++
 .../options/DataflowProfilingOptionsTest.java |  4 +-
 .../DataflowWorkerLoggingOptionsTest.java     |  4 +-
 .../beam/sdk/options/ValueProviders.java      |  4 +-
 .../apache/beam/sdk/testing/TestPipeline.java |  4 +-
 .../options/PipelineOptionsFactoryTest.java   |  2 +-
 .../options/ProxyInvocationHandlerTest.java   |  4 +-
 .../beam/sdk/options/ValueProviderTest.java   | 19 ++---
 .../beam/sdk/options/ValueProvidersTest.java  | 19 +++--
 .../beam/sdk/testing/TestPipelineTest.java    |  6 +-
 .../options/GoogleApiDebugOptionsTest.java    |  8 +-
 .../org/apache/beam/fn/harness/FnHarness.java |  5 +-
 .../runners/core/BeamFnDataReadRunner.java    |  2 -
 .../runners/core/BeamFnDataWriteRunner.java   |  2 -
 .../control/ProcessBundleHandlerTest.java     |  2 -
 .../core/BeamFnDataReadRunnerTest.java        |  2 -
 .../core/BeamFnDataWriteRunnerTest.java       |  2 -
 18 files changed, 143 insertions(+), 42 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 69b4ecd99be3..e727433eac08 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -32,6 +32,7 @@
 import static org.apache.beam.sdk.util.StringUtils.byteArrayToJsonString;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.api.services.dataflow.model.AutoscalingSettings;
 import com.google.api.services.dataflow.model.DataflowPackage;
@@ -72,6 +73,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
@@ -90,6 +92,7 @@
 import org.apache.beam.sdk.util.AppliedCombineFn;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -111,6 +114,14 @@ public class DataflowPipelineTranslator {
   private static final Logger LOG = LoggerFactory.getLogger(DataflowPipelineTranslator.class);
   private static final ObjectMapper MAPPER = new ObjectMapper();
 
+  /**
+   * Use an {@link ObjectMapper} configured with any {@link Module}s in the class path allowing
+   * for user specified configuration injection into the ObjectMapper. This supports user custom
+   * types on {@link PipelineOptions}.
+   */
+  private static final ObjectMapper MAPPER_WITH_MODULES = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+
   private static byte[] serializeWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy) {
     try {
       return WindowingStrategies.toProto(windowingStrategy).toByteArray();
@@ -303,7 +314,7 @@ public Job translate(List<DataflowPackage> packages) {
 
       try {
         environment.setSdkPipelineOptions(
-            MAPPER.readValue(MAPPER.writeValueAsBytes(options), Map.class));
+            MAPPER.readValue(MAPPER_WITH_MODULES.writeValueAsBytes(options), Map.class));
       } catch (IOException e) {
         throw new IllegalArgumentException(
             "PipelineOptions specified failed to serialize to JSON.", e);
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 41f3c925463f..a6ad8c5ca2fd 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -37,11 +37,23 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.api.services.dataflow.Dataflow;
 import com.google.api.services.dataflow.model.DataflowPackage;
 import com.google.api.services.dataflow.model.Job;
 import com.google.api.services.dataflow.model.Step;
 import com.google.api.services.dataflow.model.WorkerPool;
+import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -70,6 +82,7 @@
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.transforms.Count;
@@ -217,6 +230,76 @@ public void testSettingOfSdkPipelineOptions() throws IOException {
     assertThat(optionsMap, hasEntry("numberOfWorkerHarnessThreads", (Object) 0));
   }
 
+  /** PipelineOptions used to test auto registration of Jackson modules. */
+  public interface JacksonIncompatibleOptions extends PipelineOptions {
+    JacksonIncompatible getJacksonIncompatible();
+    void setJacksonIncompatible(JacksonIncompatible value);
+  }
+
+  /** A Jackson {@link Module} to test auto-registration of modules. */
+  @AutoService(Module.class)
+  public static class RegisteredTestModule extends SimpleModule {
+    public RegisteredTestModule() {
+      super("RegisteredTestModule");
+      setMixInAnnotation(JacksonIncompatible.class, JacksonIncompatibleMixin.class);
+    }
+  }
+
+  /** A class which Jackson does not know how to serialize/deserialize. */
+  public static class JacksonIncompatible {
+    private final String value;
+    public JacksonIncompatible(String value) {
+      this.value = value;
+    }
+  }
+
+  /** A Jackson mixin used to add annotations to other classes. */
+  @JsonDeserialize(using = JacksonIncompatibleDeserializer.class)
+  @JsonSerialize(using = JacksonIncompatibleSerializer.class)
+  public static final class JacksonIncompatibleMixin {}
+
+  /** A Jackson deserializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleDeserializer extends
+      JsonDeserializer<JacksonIncompatible> {
+
+    @Override
+    public JacksonIncompatible deserialize(JsonParser jsonParser,
+        DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+      return new JacksonIncompatible(jsonParser.readValueAs(String.class));
+    }
+  }
+
+  /** A Jackson serializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleSerializer extends JsonSerializer<JacksonIncompatible> {
+
+    @Override
+    public void serialize(JacksonIncompatible jacksonIncompatible, JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
+      jsonGenerator.writeString(jacksonIncompatible.value);
+    }
+  }
+
+  @Test
+  public void testSettingOfPipelineOptionsWithCustomUserType() throws IOException {
+    DataflowPipelineOptions options = buildPipelineOptions();
+    options.setRunner(DataflowRunner.class);
+    options.as(JacksonIncompatibleOptions.class).setJacksonIncompatible(
+        new JacksonIncompatible("userCustomTypeTest"));
+
+    Pipeline p = Pipeline.create(options);
+    p.traverseTopologically(new RecordingPipelineVisitor());
+    Job job =
+        DataflowPipelineTranslator.fromOptions(options)
+            .translate(
+                p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
+            .getJob();
+
+    Map<String, Object> sdkPipelineOptions = job.getEnvironment().getSdkPipelineOptions();
+    assertThat(sdkPipelineOptions, hasKey("options"));
+    Map<String, Object> optionsMap = (Map<String, Object>) sdkPipelineOptions.get("options");
+    assertThat(optionsMap, hasEntry("jacksonIncompatible", (Object) "userCustomTypeTest"));
+  }
+
   @Test
   public void testNetworkConfig() throws IOException {
     final String testNetwork = "test-network";
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptionsTest.java
index 4018cbbab1ef..68118a437a1d 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptionsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowProfilingOptionsTest.java
@@ -22,6 +22,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.hamcrest.Matchers;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -33,7 +34,8 @@
 @RunWith(JUnit4.class)
 public class DataflowProfilingOptionsTest {
 
-  private static final ObjectMapper MAPPER = new ObjectMapper();
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
 
   @Test
   public void testOptionsObject() throws Exception {
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java
index b463dcb86686..b1a5258eb074 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java
@@ -23,6 +23,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
 import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions.WorkerLogLevelOverrides;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -32,7 +33,8 @@
 /** Tests for {@link DataflowWorkerLoggingOptions}. */
 @RunWith(JUnit4.class)
 public class DataflowWorkerLoggingOptionsTest {
-  private static final ObjectMapper MAPPER = new ObjectMapper();
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
   @Rule public ExpectedException expectedException = ExpectedException.none();
 
   @Test
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
index d034b819f40f..e2355eeb98fa 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
@@ -24,6 +24,7 @@
 
 import java.io.IOException;
 import java.util.Map;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 
 /**
  * Utilities for working with the {@link ValueProvider} interface.
@@ -37,7 +38,8 @@ private ValueProviders() {}
    */
   public static String updateSerializedOptions(
       String serializedOptions, Map<String, String> runtimeValues) {
-    ObjectMapper mapper = new ObjectMapper();
+    ObjectMapper mapper = new ObjectMapper().registerModules(
+        ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
     ObjectNode root, options;
     try {
       root = mapper.readValue(serializedOptions, ObjectNode.class);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index ab8772bf0617..4d0cc2b36423 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -47,6 +47,7 @@
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestRule;
 import org.junit.runner.Description;
@@ -240,7 +241,8 @@ public static class PipelineRunMissingException extends RuntimeException {
 
   static final String PROPERTY_BEAM_TEST_PIPELINE_OPTIONS = "beamTestPipelineOptions";
   static final String PROPERTY_USE_DEFAULT_DUMMY_RUNNER = "beamUseDummyRunner";
-  private static final ObjectMapper MAPPER = new ObjectMapper();
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
 
   @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
   private Optional<? extends PipelineRunEnforcement> enforcement = Optional.absent();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
index 11bd7b90d3f2..76a5f18c0397 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
@@ -1689,7 +1689,7 @@ public void testRegistrationOfJacksonModulesForObjectMapper() throws Exception {
   }
 
   /** PipelineOptions used to test auto registration of Jackson modules. */
-  public interface JacksonIncompatibleOptions extends PipelineOptions {
+  interface JacksonIncompatibleOptions extends PipelineOptions {
     JacksonIncompatible getJacksonIncompatible();
     void setJacksonIncompatible(JacksonIncompatible value);
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
index 323c24ce5c77..2c43f57a40fe 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
@@ -54,6 +54,7 @@
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;
 import org.junit.Rule;
@@ -77,7 +78,8 @@ protected void before() {
     }
   };
 
-  private static final ObjectMapper MAPPER = new ObjectMapper();
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
 
   /** A test interface with some primitives and objects. */
   public interface Simple extends PipelineOptions {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
index 9369ae6957c1..e596cc106d40 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java
@@ -32,6 +32,7 @@
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -41,6 +42,8 @@
 /** Tests for {@link ValueProvider}. */
 @RunWith(JUnit4.class)
 public class ValueProviderTest {
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
   @Rule public ExpectedException expectedException = ExpectedException.none();
 
   /** A test interface. */
@@ -118,8 +121,7 @@ public void testDefaultRuntimeProvider() {
 
   @Test
   public void testNoDefaultRuntimeProviderWithOverride() throws Exception {
-    ObjectMapper mapper = new ObjectMapper();
-    TestOptions runtime = mapper.readValue(
+    TestOptions runtime = MAPPER.readValue(
       "{ \"options\": { \"foo\": \"quux\" }}", PipelineOptions.class)
       .as(TestOptions.class);
 
@@ -134,8 +136,7 @@ public void testNoDefaultRuntimeProviderWithOverride() throws Exception {
 
   @Test
   public void testDefaultRuntimeProviderWithOverride() throws Exception {
-    ObjectMapper mapper = new ObjectMapper();
-    TestOptions runtime = mapper.readValue(
+    TestOptions runtime = MAPPER.readValue(
       "{ \"options\": { \"bar\": \"quux\" }}", PipelineOptions.class)
       .as(TestOptions.class);
 
@@ -196,12 +197,11 @@ public void testOptionReturnTypeStatic() {
   public void testSerializeDeserializeNoArg() throws Exception {
     TestOptions submitOptions = PipelineOptionsFactory.as(TestOptions.class);
     assertFalse(submitOptions.getFoo().isAccessible());
-    ObjectMapper mapper = new ObjectMapper();
-    String serializedOptions = mapper.writeValueAsString(submitOptions);
+    String serializedOptions = MAPPER.writeValueAsString(submitOptions);
 
     String runnerString = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("foo", "quux"));
-    TestOptions runtime = mapper.readValue(runnerString, PipelineOptions.class)
+    TestOptions runtime = MAPPER.readValue(runnerString, PipelineOptions.class)
       .as(TestOptions.class);
 
     ValueProvider<String> vp = runtime.getFoo();
@@ -215,12 +215,11 @@ public void testSerializeDeserializeWithArg() throws Exception {
     TestOptions submitOptions = PipelineOptionsFactory.fromArgs("--foo=baz").as(TestOptions.class);
     assertEquals("baz", submitOptions.getFoo().get());
     assertTrue(submitOptions.getFoo().isAccessible());
-    ObjectMapper mapper = new ObjectMapper();
-    String serializedOptions = mapper.writeValueAsString(submitOptions);
+    String serializedOptions = MAPPER.writeValueAsString(submitOptions);
 
     String runnerString = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("foo", "quux"));
-    TestOptions runtime = mapper.readValue(runnerString, PipelineOptions.class)
+    TestOptions runtime = MAPPER.readValue(runnerString, PipelineOptions.class)
       .as(TestOptions.class);
 
     ValueProvider<String> vp = runtime.getFoo();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java
index 14f86bc279e0..dd4d55b60b16 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java
@@ -22,6 +22,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -29,6 +30,9 @@
 /** Tests for {@link ValueProviders}. */
 @RunWith(JUnit4.class)
 public class ValueProvidersTest {
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+
   /** A test interface. */
   public interface TestOptions extends PipelineOptions {
     String getString();
@@ -41,11 +45,10 @@ public interface TestOptions extends PipelineOptions {
   @Test
   public void testUpdateSerialize() throws Exception {
     TestOptions submitOptions = PipelineOptionsFactory.as(TestOptions.class);
-    ObjectMapper mapper = new ObjectMapper();
-    String serializedOptions = mapper.writeValueAsString(submitOptions);
+    String serializedOptions = MAPPER.writeValueAsString(submitOptions);
     String updatedOptions = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("string", "bar"));
-    TestOptions runtime = mapper.readValue(updatedOptions, PipelineOptions.class)
+    TestOptions runtime = MAPPER.readValue(updatedOptions, PipelineOptions.class)
       .as(TestOptions.class);
     assertEquals("bar", runtime.getString());
   }
@@ -54,11 +57,10 @@ public void testUpdateSerialize() throws Exception {
   public void testUpdateSerializeExistingValue() throws Exception {
     TestOptions submitOptions = PipelineOptionsFactory.fromArgs(
         "--string=baz", "--otherString=quux").as(TestOptions.class);
-    ObjectMapper mapper = new ObjectMapper();
-    String serializedOptions = mapper.writeValueAsString(submitOptions);
+    String serializedOptions = MAPPER.writeValueAsString(submitOptions);
     String updatedOptions = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.of("string", "bar"));
-    TestOptions runtime = mapper.readValue(updatedOptions, PipelineOptions.class)
+    TestOptions runtime = MAPPER.readValue(updatedOptions, PipelineOptions.class)
       .as(TestOptions.class);
     assertEquals("bar", runtime.getString());
     assertEquals("quux", runtime.getOtherString());
@@ -67,11 +69,10 @@ public void testUpdateSerializeExistingValue() throws Exception {
   @Test
   public void testUpdateSerializeEmptyUpdate() throws Exception {
     TestOptions submitOptions = PipelineOptionsFactory.as(TestOptions.class);
-    ObjectMapper mapper = new ObjectMapper();
-    String serializedOptions = mapper.writeValueAsString(submitOptions);
+    String serializedOptions = MAPPER.writeValueAsString(submitOptions);
     String updatedOptions = ValueProviders.updateSerializedOptions(
       serializedOptions, ImmutableMap.<String, String>of());
-    TestOptions runtime = mapper.readValue(updatedOptions, PipelineOptions.class)
+    TestOptions runtime = MAPPER.readValue(updatedOptions, PipelineOptions.class)
       .as(TestOptions.class);
     assertNull(runtime.getString());
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java
index 04005c5893b8..05abb59c597e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java
@@ -40,6 +40,7 @@
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.apache.beam.sdk.values.PCollection;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
@@ -61,6 +62,8 @@
   TestPipelineTest.TestPipelineEnforcementsTest.WithCrashingPipelineRunner.class
 })
 public class TestPipelineTest implements Serializable {
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
 
   /** Tests related to the creation of a {@link TestPipeline}. */
   @RunWith(JUnit4.class)
@@ -85,9 +88,8 @@ public void testCreationNotAsTestRule() {
 
     @Test
     public void testCreationOfPipelineOptions() throws Exception {
-      ObjectMapper mapper = new ObjectMapper();
       String stringOptions =
-          mapper.writeValueAsString(
+          MAPPER.writeValueAsString(
               new String[] {
                 "--runner=org.apache.beam.sdk.testing.CrashingRunner"
               });
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
index 67d588029f9d..68a29e68e74a 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
@@ -28,6 +28,7 @@
 import org.apache.beam.sdk.extensions.gcp.options.GoogleApiDebugOptions.GoogleApiTracer;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.util.Transport;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -35,6 +36,8 @@
 /** Tests for {@link GoogleApiDebugOptions}. */
 @RunWith(JUnit4.class)
 public class GoogleApiDebugOptionsTest {
+  private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
+      ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
   private static final String STORAGE_GET_TRACE =
       "--googleApiTrace={\"Objects.Get\":\"GetTraceDestination\"}";
   private static final String STORAGE_GET_AND_LIST_TRACE =
@@ -139,9 +142,8 @@ public void testMatchingAgainstRequestType() throws Exception {
   @Test
   public void testDeserializationAndSerializationOfGoogleApiTracer() throws Exception {
     String serializedValue = "{\"Api\":\"Token\"}";
-    ObjectMapper objectMapper = new ObjectMapper();
     assertEquals(serializedValue,
-        objectMapper.writeValueAsString(
-            objectMapper.readValue(serializedValue, GoogleApiTracer.class)));
+        MAPPER.writeValueAsString(
+            MAPPER.readValue(serializedValue, GoogleApiTracer.class)));
   }
 }
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
index d5879864db84..24f826c5c522 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
@@ -34,6 +34,7 @@
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -72,7 +73,9 @@ public static void main(String[] args) throws Exception {
     System.out.format("Control location %s%n", System.getenv(CONTROL_API_SERVICE_DESCRIPTOR));
     System.out.format("Pipeline options %s%n", System.getenv(PIPELINE_OPTIONS));
 
-    PipelineOptions options = new ObjectMapper().readValue(
+    ObjectMapper objectMapper = new ObjectMapper().registerModules(
+        ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+    PipelineOptions options = objectMapper.readValue(
         System.getenv(PIPELINE_OPTIONS), PipelineOptions.class);
 
     BeamFnApi.ApiServiceDescriptor loggingApiServiceDescriptor =
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
index 805d480e7dae..7c4a5e8fa272 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
@@ -18,7 +18,6 @@
 
 package org.apache.beam.runners.core;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
@@ -47,7 +46,6 @@
  */
 public class BeamFnDataReadRunner<OutputT> {
   private static final Logger LOG = LoggerFactory.getLogger(BeamFnDataReadRunner.class);
-  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
   private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
   private final Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
index 0ba09e392a26..3a11def393fc 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
@@ -18,7 +18,6 @@
 
 package org.apache.beam.runners.core;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.util.function.Supplier;
 import org.apache.beam.fn.harness.data.BeamFnDataClient;
@@ -38,7 +37,6 @@
  * For each request, call {@link #registerForOutput()} to start and call {@link #close()} to finish.
  */
 public class BeamFnDataWriteRunner<InputT> {
-  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
   private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
   private final BeamFnApi.Target outputTarget;
   private final Coder<WindowedValue<InputT>> coder;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 59872678b3cd..654f98988cc0 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -33,7 +33,6 @@
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
@@ -92,7 +91,6 @@
 /** Tests for {@link ProcessBundleHandler}. */
 @RunWith(JUnit4.class)
 public class ProcessBundleHandlerTest {
-  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
   private static final Coder<WindowedValue<String>> STRING_CODER =
       WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
   private static final String LONG_CODER_SPEC_ID = "998L";
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
index 0d036fe4edc7..04a3615dde58 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
@@ -27,7 +27,6 @@
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Uninterruptibles;
@@ -65,7 +64,6 @@
 /** Tests for {@link BeamFnDataReadRunner}. */
 @RunWith(JUnit4.class)
 public class BeamFnDataReadRunnerTest {
-  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
   private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
       .setApiServiceDescriptor(BeamFnApi.ApiServiceDescriptor.getDefaultInstance()).build();
   private static final BeamFnApi.FunctionSpec FUNCTION_SPEC = BeamFnApi.FunctionSpec.newBuilder()
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
index 50fee7a98082..9e50cd0e6212 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
@@ -28,7 +28,6 @@
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.protobuf.Any;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -53,7 +52,6 @@
 /** Tests for {@link BeamFnDataWriteRunner}. */
 @RunWith(JUnit4.class)
 public class BeamFnDataWriteRunnerTest {
-  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
   private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
       .setApiServiceDescriptor(BeamFnApi.ApiServiceDescriptor.getDefaultInstance()).build();
   private static final BeamFnApi.FunctionSpec FUNCTION_SPEC = BeamFnApi.FunctionSpec.newBuilder()

From f53e5d43d58c79ab9f3d04e112e6f05ad9dfe42f Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Wed, 3 May 2017 17:12:20 -0700
Subject: [PATCH 011/387] [BEAM-2165] Update Flink to support
 serializing/deserializing custom user types configured via Jackson modules

---
 runners/flink/pom.xml                         |  5 ++
 .../utils/SerializedPipelineOptions.java      | 16 +++-
 .../runners/flink/PipelineOptionsTest.java    | 87 +++++++++++++++++++
 3 files changed, 106 insertions(+), 2 deletions(-)

diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index eb2b005ddfd3..41224545cf4f 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -255,6 +255,11 @@
       <artifactId>jackson-annotations</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
index 2256bb1df6c8..f717fd76c5e6 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
@@ -20,6 +20,7 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -27,6 +28,7 @@
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 
 /**
  * Encapsulates the PipelineOptions in serialized form to ship them to the cluster.
@@ -42,7 +44,7 @@ public SerializedPipelineOptions(PipelineOptions options) {
     checkNotNull(options, "PipelineOptions must not be null.");
 
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-      new ObjectMapper().writeValue(baos, options);
+      createMapper().writeValue(baos, options);
       this.serializedOptions = baos.toByteArray();
     } catch (Exception e) {
       throw new RuntimeException("Couldn't serialize PipelineOptions.", e);
@@ -53,7 +55,7 @@ public SerializedPipelineOptions(PipelineOptions options) {
   public PipelineOptions getPipelineOptions() {
     if (pipelineOptions == null) {
       try {
-        pipelineOptions = new ObjectMapper().readValue(serializedOptions, PipelineOptions.class);
+        pipelineOptions = createMapper().readValue(serializedOptions, PipelineOptions.class);
 
         IOChannelUtils.registerIOFactoriesAllowOverride(pipelineOptions);
         FileSystems.setDefaultConfigInWorkers(pipelineOptions);
@@ -64,4 +66,14 @@ public PipelineOptions getPipelineOptions() {
 
     return pipelineOptions;
   }
+
+  /**
+   * Use an {@link ObjectMapper} configured with any {@link Module}s in the class path allowing
+   * for user specified configuration injection into the ObjectMapper. This supports user custom
+   * types on {@link PipelineOptions}.
+   */
+  private static ObjectMapper createMapper() {
+    return new ObjectMapper().registerModules(
+        ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+  }
 }
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
index 23740a135944..7519dbf9d5c1 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
@@ -23,6 +23,23 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.auto.service.AutoService;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.util.Collections;
 import java.util.HashMap;
 import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
@@ -200,4 +217,74 @@ public void processElement(ProcessContext c) throws Exception {
           c.getPipelineOptions().as(MyOptions.class).getTestOption());
     }
   }
+
+  /** PipelineOptions used to test auto registration of Jackson modules. */
+  public interface JacksonIncompatibleOptions extends PipelineOptions {
+    JacksonIncompatible getJacksonIncompatible();
+    void setJacksonIncompatible(JacksonIncompatible value);
+  }
+
+  /** A Jackson {@link Module} to test auto-registration of modules. */
+  @AutoService(Module.class)
+  public static class RegisteredTestModule extends SimpleModule {
+    public RegisteredTestModule() {
+      super("RegisteredTestModule");
+      setMixInAnnotation(JacksonIncompatible.class, JacksonIncompatibleMixin.class);
+    }
+  }
+
+  /** A class which Jackson does not know how to serialize/deserialize. */
+  public static class JacksonIncompatible {
+    private final String value;
+    public JacksonIncompatible(String value) {
+      this.value = value;
+    }
+  }
+
+  /** A Jackson mixin used to add annotations to other classes. */
+  @JsonDeserialize(using = JacksonIncompatibleDeserializer.class)
+  @JsonSerialize(using = JacksonIncompatibleSerializer.class)
+  public static final class JacksonIncompatibleMixin {}
+
+  /** A Jackson deserializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleDeserializer extends
+      JsonDeserializer<JacksonIncompatible> {
+
+    @Override
+    public JacksonIncompatible deserialize(JsonParser jsonParser,
+        DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+      return new JacksonIncompatible(jsonParser.readValueAs(String.class));
+    }
+  }
+
+  /** A Jackson serializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleSerializer extends JsonSerializer<JacksonIncompatible> {
+
+    @Override
+    public void serialize(JacksonIncompatible jacksonIncompatible, JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
+      jsonGenerator.writeString(jacksonIncompatible.value);
+    }
+  }
+
+  @Test
+  public void testSerializingPipelineOptionsWithCustomUserType() throws Exception {
+    String expectedValue = "testValue";
+    PipelineOptions options = PipelineOptionsFactory
+        .fromArgs("--jacksonIncompatible=\"" + expectedValue + "\"")
+        .as(JacksonIncompatibleOptions.class);
+    SerializedPipelineOptions context = new SerializedPipelineOptions(options);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    try (ObjectOutputStream outputStream = new ObjectOutputStream(baos)) {
+      outputStream.writeObject(context);
+    }
+    try (ObjectInputStream inputStream =
+        new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()))) {
+      SerializedPipelineOptions copy = (SerializedPipelineOptions) inputStream.readObject();
+      assertEquals(expectedValue,
+          copy.getPipelineOptions().as(JacksonIncompatibleOptions.class)
+              .getJacksonIncompatible().value);
+    }
+  }
 }

From cc654f02e8670ea789aee67508c569e7547ef11f Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Wed, 3 May 2017 13:48:07 -0700
Subject: [PATCH 012/387] [BEAM-1871] Migrate ReleaseInfo away from Google API
 client GenericJson

---
 .../beam/runners/dataflow/DataflowRunner.java |  2 +-
 .../org/apache/beam/sdk/util/ReleaseInfo.java | 70 +++++++++++--------
 2 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 9e5a2fb114d6..2b54ba74e7d3 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -542,7 +542,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
         "Unable to submit a job to the Dataflow service with unset version ${pom.version}");
     System.out.println("Dataflow SDK version: " + version);
 
-    newJob.getEnvironment().setUserAgent(releaseInfo);
+    newJob.getEnvironment().setUserAgent((Map) releaseInfo.getProperties());
     // The Dataflow Service may write to the temporary directory directly, so
     // must be verified.
     if (!isNullOrEmpty(options.getGcpTempLocation())) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java
index eeac5571c316..c7e24d26d315 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java
@@ -17,27 +17,24 @@
  */
 package org.apache.beam.sdk.util;
 
-import com.google.api.client.json.GenericJson;
-import com.google.api.client.util.Key;
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Serializable;
+import java.util.Map;
 import java.util.Properties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Utilities for working with release information.
+ * Properties pertaining to this release of Apache Beam.
+ *
+ * <p>Properties will always include a name and version.
  */
-public final class ReleaseInfo extends GenericJson {
-  private static final Logger LOG = LoggerFactory.getLogger(ReleaseInfo.class);
-
-  private static final String PROPERTIES_PATH =
-      "/org/apache/beam/sdk/sdk.properties";
-
-  private static class LazyInit {
-    private static final ReleaseInfo INSTANCE =
-        new ReleaseInfo(PROPERTIES_PATH);
-  }
+@AutoValue
+public abstract class ReleaseInfo implements Serializable {
+  private static final String PROPERTIES_PATH = "/org/apache/beam/sdk/sdk.properties";
 
   /**
    * Returns an instance of {@link ReleaseInfo}.
@@ -46,35 +43,46 @@ public static ReleaseInfo getReleaseInfo() {
     return LazyInit.INSTANCE;
   }
 
-  @Key private String name = "Apache Beam SDK for Java";
-  @Key private String version = "Unknown";
+  /**
+   * Returns an immutable map of all properties pertaining to this release.
+   */
+  public abstract Map<String, String> getProperties();
 
   /** Provides the SDK name. */
   public String getName() {
-    return name;
+    return getProperties().get("name");
   }
 
   /** Provides the SDK version. */
   public String getVersion() {
-    return version;
+    return getProperties().get("version");
   }
 
-  private ReleaseInfo(String resourcePath) {
-    Properties properties = new Properties();
+  /////////////////////////////////////////////////////////////////////////
+  private static final Logger LOG = LoggerFactory.getLogger(ReleaseInfo.class);
+  private static final String DEFAULT_NAME = "Apache Beam SDK for Java";
+  private static final String DEFAULT_VERSION = "Unknown";
 
-    try (InputStream in = ReleaseInfo.class.getResourceAsStream(PROPERTIES_PATH)) {
-      if (in == null) {
-        LOG.warn("Beam properties resource not found: {}", resourcePath);
-        return;
+  private static class LazyInit {
+    private static final ReleaseInfo INSTANCE;
+    static {
+      Properties properties = new Properties();
+      try (InputStream in = ReleaseInfo.class.getResourceAsStream(PROPERTIES_PATH)) {
+        if (in == null) {
+          LOG.warn("Beam properties resource not found: {}", PROPERTIES_PATH);
+        } else {
+          properties.load(in);
+        }
+      } catch (IOException e) {
+        LOG.warn("Error loading Beam properties resource: ", e);
       }
-
-      properties.load(in);
-    } catch (IOException e) {
-      LOG.warn("Error loading Beam properties resource: ", e);
-    }
-
-    for (String name : properties.stringPropertyNames()) {
-      put(name, properties.getProperty(name));
+      if (!properties.containsKey("name")) {
+        properties.setProperty("name", DEFAULT_NAME);
+      }
+      if (!properties.containsKey("version")) {
+        properties.setProperty("version", DEFAULT_VERSION);
+      }
+      INSTANCE = new AutoValue_ReleaseInfo(ImmutableMap.copyOf((Map) properties));
     }
   }
 }

From 8a2dcdb6f9d4839c864a2c46c4b5254d0c7d4760 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 11:52:02 -0700
Subject: [PATCH 013/387] DataflowRunner: integration test GCP-IO

Triggered under `-DskipITs=false -Pdataflow-runner`
---
 runners/google-cloud-dataflow-java/pom.xml | 50 ++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 30ef84d9a461..b0ae757b51aa 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -79,6 +79,49 @@
         </pluginManagement>
       </build>
     </profile>
+
+    <!-- A profile that runs the integration tests in the DataflowRunner. -->
+    <profile>
+      <id>dataflow-runner</id>
+      <build>
+        <pluginManagement>
+          <plugins>
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-failsafe-plugin</artifactId>
+              <executions>
+                <execution>
+                  <id>integration-test</id>
+                  <goals>
+                    <goal>integration-test</goal>
+                    <goal>verify</goal>
+                  </goals>
+                  <configuration>
+                    <useManifestOnlyJar>false</useManifestOnlyJar>
+                    <failIfNoTests>true</failIfNoTests>
+                    <parallel>all</parallel>
+                    <threadCount>4</threadCount>
+                    <dependenciesToScan>
+                      <dependency>org.apache.beam:beam-sdks-java-io-google-cloud-platform</dependency>
+                    </dependenciesToScan>
+                    <systemPropertyVariables>
+                      <beamTestPipelineOptions>${integrationTestPipelineOptions}</beamTestPipelineOptions>
+                    </systemPropertyVariables>
+                  </configuration>
+                </execution>
+              </executions>
+            </plugin>
+          </plugins>
+        </pluginManagement>
+
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <build>
@@ -216,6 +259,13 @@
       <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-runners-core-construction-java</artifactId>

From 1671708340fb9fc57cdc91c3bbacdff3ae6af4af Mon Sep 17 00:00:00 2001
From: "yangping.wu" <yangping.wu@qunar.com>
Date: Thu, 4 May 2017 14:04:08 +0800
Subject: [PATCH 014/387] [BEAM-1491]Identify HADOOP_CONF_DIR(or YARN_CONF_DIR)
 environment variables

---
 sdks/java/io/hadoop-file-system/pom.xml       |   6 +
 .../sdk/io/hdfs/HadoopFileSystemOptions.java  |  73 +++++++++-
 .../io/hdfs/HadoopFileSystemOptionsTest.java  | 125 ++++++++++++++++++
 3 files changed, 197 insertions(+), 7 deletions(-)

diff --git a/sdks/java/io/hadoop-file-system/pom.xml b/sdks/java/io/hadoop-file-system/pom.xml
index 3ec98482cf74..562277eeb116 100644
--- a/sdks/java/io/hadoop-file-system/pom.xml
+++ b/sdks/java/io/hadoop-file-system/pom.xml
@@ -186,6 +186,12 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java
index 31250bcd5868..45f43e23c80d 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java
@@ -17,12 +17,22 @@
  */
 package org.apache.beam.sdk.io.hdfs;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import java.io.File;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * {@link PipelineOptions} which encapsulate {@link Configuration Hadoop Configuration}
@@ -30,20 +40,69 @@
  */
 public interface HadoopFileSystemOptions extends PipelineOptions {
   @Description("A list of Hadoop configurations used to configure zero or more Hadoop filesystems. "
-      + "To specify on the command-line, represent the value as a JSON list of JSON maps, where "
-      + "each map represents the entire configuration for a single Hadoop filesystem. For example "
-      + "--hdfsConfiguration='[{\"fs.default.name\": \"hdfs://localhost:9998\", ...},"
+      + "By default, Hadoop configuration is loaded from 'core-site.xml' and 'hdfs-site.xml' "
+      + "based upon the HADOOP_CONF_DIR and YARN_CONF_DIR environment variables. "
+      + "To specify configuration on the command-line, represent the value as a JSON list of JSON "
+      + "maps, where each map represents the entire configuration for a single Hadoop filesystem. "
+      + "For example --hdfsConfiguration='[{\"fs.default.name\": \"hdfs://localhost:9998\", ...},"
       + "{\"fs.default.name\": \"s3a://\", ...},...]'")
   @Default.InstanceFactory(ConfigurationLocator.class)
   List<Configuration> getHdfsConfiguration();
   void setHdfsConfiguration(List<Configuration> value);
 
   /** A {@link DefaultValueFactory} which locates a Hadoop {@link Configuration}. */
-  class ConfigurationLocator implements DefaultValueFactory<Configuration> {
+  class ConfigurationLocator implements DefaultValueFactory<List<Configuration>> {
+    private static final Logger LOG = LoggerFactory.getLogger(ConfigurationLocator.class);
     @Override
-    public Configuration create(PipelineOptions options) {
-      // TODO: Find default configuration to use
-      return null;
+    public List<Configuration> create(PipelineOptions options) {
+      // Find default configuration when HADOOP_CONF_DIR or YARN_CONF_DIR is set.
+      List<Configuration> configurationList = readConfigurationFromHadoopYarnConfigDirs();
+      return configurationList.size() > 0 ? configurationList : null;
+    }
+
+    private List<Configuration> readConfigurationFromHadoopYarnConfigDirs() {
+      List<Configuration> configurationList = Lists.newArrayList();
+
+      /*
+      * If we find a configuration in HADOOP_CONF_DIR and YARN_CONF_DIR,
+      * we should be returning them both separately.
+      *
+      * Also, ensure that we only load one configuration if both
+      * HADOOP_CONF_DIR and YARN_CONF_DIR point to the same location.
+      */
+      Set<String> confDirs = Sets.newHashSet();
+      for (String confDir : Lists.newArrayList("HADOOP_CONF_DIR", "YARN_CONF_DIR")) {
+        if (getEnvironment().containsKey(confDir)) {
+          String hadoopConfDir = getEnvironment().get(confDir);
+          if (!Strings.isNullOrEmpty(hadoopConfDir)) {
+            confDirs.add(hadoopConfDir);
+          }
+        }
+      }
+
+      for (String confDir : confDirs) {
+        if (new File(confDir).exists()) {
+          Configuration conf = new Configuration(false);
+          boolean confLoaded = false;
+          for (String confName : Lists.newArrayList("core-site.xml", "hdfs-site.xml")) {
+            File confFile = new File(confDir, confName);
+            if (confFile.exists()) {
+              LOG.debug("Adding {} to hadoop configuration", confFile.getAbsolutePath());
+              conf.addResource(new Path(confFile.getAbsolutePath()));
+              confLoaded = true;
+            }
+          }
+          if (confLoaded) {
+            configurationList.add(conf);
+          }
+        }
+      }
+      return configurationList;
+    }
+
+    @VisibleForTesting
+    Map<String, String> getEnvironment() {
+      return System.getenv();
     }
   }
 }
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java
index 634528b8f33f..c99fc5b4ca0f 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java
@@ -18,13 +18,26 @@
 package org.apache.beam.sdk.io.hdfs;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.common.io.Files;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.AbstractMap;
+import java.util.List;
 import java.util.Map;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.hamcrest.Matchers;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -33,6 +46,8 @@
  */
 @RunWith(JUnit4.class)
 public class HadoopFileSystemOptionsTest {
+  @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
+
   @Test
   public void testParsingHdfsConfiguration() {
     HadoopFileSystemOptions options = PipelineOptionsFactory.fromArgs(
@@ -45,4 +60,114 @@ public void testParsingHdfsConfiguration() {
     assertThat(options.getHdfsConfiguration().get(1), Matchers.<Map.Entry<String, String>>contains(
         new AbstractMap.SimpleEntry("propertyB", "B")));
   }
+
+  @Test
+  public void testDefaultUnsetEnvHdfsConfiguration() {
+    HadoopFileSystemOptions.ConfigurationLocator projectFactory =
+            spy(new HadoopFileSystemOptions.ConfigurationLocator());
+    when(projectFactory.getEnvironment()).thenReturn(ImmutableMap.<String, String>of());
+    assertNull(projectFactory.create(PipelineOptionsFactory.create()));
+  }
+
+  @Test
+  public void testDefaultJustSetHadoopConfDirConfiguration() throws IOException {
+    Files.write(createPropertyData("A"),
+        tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8);
+    Files.write(createPropertyData("B"),
+        tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8);
+    HadoopFileSystemOptions.ConfigurationLocator configurationLocator =
+            spy(new HadoopFileSystemOptions.ConfigurationLocator());
+    Map<String, String> environment = Maps.newHashMap();
+    environment.put("HADOOP_CONF_DIR", tmpFolder.getRoot().getAbsolutePath());
+    when(configurationLocator.getEnvironment()).thenReturn(environment);
+
+    List<Configuration> configurationList =
+        configurationLocator.create(PipelineOptionsFactory.create());
+    assertEquals(1, configurationList.size());
+    assertThat(configurationList.get(0).get("propertyA"), Matchers.equalTo("A"));
+    assertThat(configurationList.get(0).get("propertyB"), Matchers.equalTo("B"));
+  }
+
+  @Test
+  public void testDefaultJustSetYarnConfDirConfiguration() throws IOException {
+    Files.write(createPropertyData("A"),
+        tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8);
+    Files.write(createPropertyData("B"),
+        tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8);
+    HadoopFileSystemOptions.ConfigurationLocator configurationLocator =
+            spy(new HadoopFileSystemOptions.ConfigurationLocator());
+    Map<String, String> environment = Maps.newHashMap();
+    environment.put("YARN_CONF_DIR", tmpFolder.getRoot().getAbsolutePath());
+    when(configurationLocator.getEnvironment()).thenReturn(environment);
+
+    List<Configuration> configurationList =
+        configurationLocator.create(PipelineOptionsFactory.create());
+    assertEquals(1, configurationList.size());
+    assertThat(configurationList.get(0).get("propertyA"), Matchers.equalTo("A"));
+    assertThat(configurationList.get(0).get("propertyB"), Matchers.equalTo("B"));
+  }
+
+  @Test
+  public void testDefaultSetYarnConfDirAndHadoopConfDirAndSameConfiguration() throws IOException {
+    Files.write(createPropertyData("A"),
+        tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8);
+    Files.write(createPropertyData("B"),
+        tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8);
+    HadoopFileSystemOptions.ConfigurationLocator configurationLocator =
+            spy(new HadoopFileSystemOptions.ConfigurationLocator());
+    Map<String, String> environment = Maps.newHashMap();
+    environment.put("YARN_CONF_DIR", tmpFolder.getRoot().getAbsolutePath());
+    environment.put("HADOOP_CONF_DIR", tmpFolder.getRoot().getAbsolutePath());
+    when(configurationLocator.getEnvironment()).thenReturn(environment);
+
+    List<Configuration> configurationList =
+        configurationLocator.create(PipelineOptionsFactory.create());
+    assertEquals(1, configurationList.size());
+    assertThat(configurationList.get(0).get("propertyA"), Matchers.equalTo("A"));
+    assertThat(configurationList.get(0).get("propertyB"), Matchers.equalTo("B"));
+  }
+
+  @Test
+  public void testDefaultSetYarnConfDirAndHadoopConfDirNotSameConfiguration() throws IOException {
+    File hadoopConfDir = tmpFolder.newFolder("hadoop");
+    File yarnConfDir = tmpFolder.newFolder("yarn");
+    Files.write(createPropertyData("A"),
+        new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8);
+    Files.write(createPropertyData("B"),
+        new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8);
+    Files.write(createPropertyData("C"),
+        new File(yarnConfDir, "core-site.xml"), StandardCharsets.UTF_8);
+    Files.write(createPropertyData("D"),
+        new File(yarnConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8);
+    HadoopFileSystemOptions.ConfigurationLocator configurationLocator =
+              spy(new HadoopFileSystemOptions.ConfigurationLocator());
+    Map<String, String> environment = Maps.newHashMap();
+    environment.put("YARN_CONF_DIR", hadoopConfDir.getAbsolutePath());
+    environment.put("HADOOP_CONF_DIR", yarnConfDir.getAbsolutePath());
+    when(configurationLocator.getEnvironment()).thenReturn(environment);
+
+    List<Configuration> configurationList =
+        configurationLocator.create(PipelineOptionsFactory.create());
+    assertEquals(2, configurationList.size());
+    int hadoopConfIndex = configurationList.get(0).get("propertyA") != null ? 0 : 1;
+    assertThat(
+        configurationList.get(hadoopConfIndex).get("propertyA"), Matchers.equalTo("A"));
+    assertThat(
+        configurationList.get(hadoopConfIndex).get("propertyB"), Matchers.equalTo("B"));
+    assertThat(
+        configurationList.get(1 - hadoopConfIndex).get("propertyC"), Matchers.equalTo("C"));
+    assertThat(
+        configurationList.get(1 - hadoopConfIndex).get("propertyD"), Matchers.equalTo("D"));
+  }
+
+  private static String createPropertyData(String property) {
+    return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+        + "<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n"
+        + "<configuration>\n"
+        + "    <property>\n"
+        + "        <name>property" + property + "</name>\n"
+        + "        <value>" + property + "</value>\n"
+        + "    </property>\n"
+        + "</configuration>";
+  }
 }

From fba3d87ffec08f84c8be08ee16942b13364da2d9 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@google.com>
Date: Wed, 3 May 2017 14:56:37 -0700
Subject: [PATCH 015/387] Split Coder's encode/decode methods into two methods
 depending on context.

This allows the outer context to be marked deprecated.  A follow-up PR will
remove the old method once all consumers have been updated.
---
 .../org/apache/beam/sdk/coders/Coder.java     | 72 +++++++++++++++++++
 .../beam/sdk/coders/StructuredCoder.java      | 61 ++++++++++++++++
 2 files changed, 133 insertions(+)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index 8ba8ad3e6c23..c923719eb025 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -59,6 +59,7 @@
  */
 public interface Coder<T> extends Serializable {
   /** The context in which encoding or decoding is being done. */
+  @Deprecated
   class Context {
     /**
      * The outer context: the value being encoded or decoded takes
@@ -110,6 +111,28 @@ public String toString() {
     }
   }
 
+  /**
+   * Encodes the given value of type {@code T} onto the given output stream.
+   *
+   * @throws IOException if writing to the {@code OutputStream} fails
+   * for some reason
+   * @throws CoderException if the value could not be encoded for some reason
+   */
+  void encode(T value, OutputStream outStream)
+      throws CoderException, IOException;
+
+  /**
+   * Encodes the given value of type {@code T} onto the given output stream
+   * in the outer context.
+   *
+   * @throws IOException if writing to the {@code OutputStream} fails
+   * for some reason
+   * @throws CoderException if the value could not be encoded for some reason
+   */
+  @Deprecated
+  void encodeOuter(T value, OutputStream outStream)
+      throws CoderException, IOException;
+
   /**
    * Encodes the given value of type {@code T} onto the given output stream
    * in the given context.
@@ -118,6 +141,7 @@ public String toString() {
    * for some reason
    * @throws CoderException if the value could not be encoded for some reason
    */
+  @Deprecated
   void encode(T value, OutputStream outStream, Context context)
       throws CoderException, IOException;
 
@@ -129,6 +153,28 @@ void encode(T value, OutputStream outStream, Context context)
    * for some reason
    * @throws CoderException if the value could not be decoded for some reason
    */
+  T decode(InputStream inStream) throws CoderException, IOException;
+
+  /**
+   * Decodes a value of type {@code T} from the given input stream in
+   * the outer context.  Returns the decoded value.
+   *
+   * @throws IOException if reading from the {@code InputStream} fails
+   * for some reason
+   * @throws CoderException if the value could not be decoded for some reason
+   */
+  @Deprecated
+  T decodeOuter(InputStream inStream) throws CoderException, IOException;
+
+  /**
+   * Decodes a value of type {@code T} from the given input stream in
+   * the given context.  Returns the decoded value.
+   *
+   * @throws IOException if reading from the {@code InputStream} fails
+   * for some reason
+   * @throws CoderException if the value could not be decoded for some reason
+   */
+  @Deprecated
   T decode(InputStream inStream, Context context)
       throws CoderException, IOException;
 
@@ -200,6 +246,19 @@ T decode(InputStream inStream, Context context)
    * {@link org.apache.beam.sdk.runners.PipelineRunner}
    * implementations.
    */
+  boolean isRegisterByteSizeObserverCheap(T value);
+
+  /**
+   * Returns whether {@link #registerByteSizeObserver} cheap enough to
+   * call for every element, that is, if this {@code Coder} can
+   * calculate the byte size of the element to be coded in roughly
+   * constant time (or lazily).
+   *
+   * <p>Not intended to be called by user code, but instead by
+   * {@link org.apache.beam.sdk.runners.PipelineRunner}
+   * implementations.
+   */
+  @Deprecated
   boolean isRegisterByteSizeObserverCheap(T value, Context context);
 
   /**
@@ -210,6 +269,19 @@ T decode(InputStream inStream, Context context)
    * {@link org.apache.beam.sdk.runners.PipelineRunner}
    * implementations.
    */
+  void registerByteSizeObserver(
+      T value, ElementByteSizeObserver observer)
+      throws Exception;
+
+  /**
+   * Notifies the {@code ElementByteSizeObserver} about the byte size
+   * of the encoded value using this {@code Coder}.
+   *
+   * <p>Not intended to be called by user code, but instead by
+   * {@link org.apache.beam.sdk.runners.PipelineRunner}
+   * implementations.
+   */
+  @Deprecated
   void registerByteSizeObserver(
       T value, ElementByteSizeObserver observer, Context context)
       throws Exception;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
index 0cd53b0573c8..cc39429a8e20 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
@@ -20,6 +20,9 @@
 import com.google.common.io.ByteStreams;
 import com.google.common.io.CountingOutputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -98,6 +101,58 @@ public String toString() {
     return builder.toString();
   }
 
+  public void encode(T value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Coder.Context.NESTED);
+  }
+
+  @Deprecated
+  public void encodeOuter(T value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Coder.Context.OUTER);
+  }
+
+  @Deprecated
+  public void encode(T value, OutputStream outStream, Coder.Context context)
+      throws CoderException, IOException {
+    if (context == Coder.Context.NESTED) {
+      encode(value, outStream);
+    } else {
+      encodeOuter(value, outStream);
+    }
+  }
+
+  public T decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Coder.Context.NESTED);
+  }
+
+  @Deprecated
+  public T decodeOuter(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Coder.Context.OUTER);
+  }
+
+  @Deprecated
+  public T decode(InputStream inStream, Coder.Context context)
+      throws CoderException, IOException {
+    if (context == Coder.Context.NESTED) {
+      return decode(inStream);
+    } else {
+      return decodeOuter(inStream);
+    }
+  }
+
+  /**
+   * {@inheritDoc}
+   *
+   * @return {@code false} unless it is overridden. {@link StructuredCoder#registerByteSizeObserver}
+   *         invokes {@link #getEncodedElementByteSize} which requires re-encoding an element
+   *         unless it is overridden. This is considered expensive.
+   */
+  @Override
+  public boolean isRegisterByteSizeObserverCheap(T value) {
+    return isRegisterByteSizeObserverCheap(value, Context.NESTED);
+  }
+
   /**
    * {@inheritDoc}
    *
@@ -124,6 +179,12 @@ protected long getEncodedElementByteSize(T value, Context context)
     }
   }
 
+  @Override
+  public void registerByteSizeObserver(T value, ElementByteSizeObserver observer)
+      throws Exception {
+    registerByteSizeObserver(value, observer, Context.NESTED);
+  }
+
   /**
    * {@inheritDoc}
    *

From 690ec3b1f7b6ce9caaa7b9e401878e136f44bc50 Mon Sep 17 00:00:00 2001
From: bchambers <bchambers@google.com>
Date: Wed, 3 May 2017 16:40:09 -0700
Subject: [PATCH 016/387] [BEAM-2162] Add logging to long BigQuery jobs

---
 .../io/gcp/bigquery/BigQueryServicesImpl.java | 20 ++++++++++++++++++-
 .../io/gcp/bigquery/BigQuerySourceBase.java   |  2 ++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index 8e395f0da7f1..b348abdd22df 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -69,6 +69,7 @@
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
 import org.apache.beam.sdk.util.Transport;
 import org.joda.time.Duration;
+import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -80,6 +81,9 @@ class BigQueryServicesImpl implements BigQueryServices {
 
   private static final Logger LOG = LoggerFactory.getLogger(BigQueryServicesImpl.class);
 
+  // How frequently to log while polling.
+  private static final Duration POLLING_LOG_GAP = Duration.standardMinutes(10);
+
   // The maximum number of retries to execute a BigQuery RPC.
   private static final int MAX_RPC_RETRIES = 9;
 
@@ -219,7 +223,8 @@ static void startJob(
       do {
         try {
           client.jobs().insert(jobRef.getProjectId(), job).execute();
-          LOG.info("Started BigQuery job: {}.", jobRef);
+          LOG.info("Started BigQuery job: {}.\n{}", jobRef,
+              formatBqStatusCommand(jobRef.getProjectId(), jobRef.getJobId()));
           return; // SUCCEEDED
         } catch (GoogleJsonResponseException e) {
           if (errorExtractor.itemAlreadyExists(e)) {
@@ -257,6 +262,7 @@ Job pollJob(
         JobReference jobRef,
         Sleeper sleeper,
         BackOff backoff) throws InterruptedException {
+      Instant nextLog = Instant.now().plus(POLLING_LOG_GAP);
       do {
         try {
           Job job = client.jobs().get(jobRef.getProjectId(), jobRef.getJobId()).execute();
@@ -265,6 +271,13 @@ Job pollJob(
             return job;
           }
           // The job is not DONE, wait longer and retry.
+          if (Instant.now().isAfter(nextLog)) {
+            LOG.info("Still waiting for BigQuery job {}\n{}",
+                jobRef.getJobId(),
+                formatBqStatusCommand(
+                    jobRef.getProjectId(), jobRef.getJobId()));
+            nextLog = Instant.now().plus(POLLING_LOG_GAP);
+          }
         } catch (IOException e) {
           // ignore and retry
           LOG.info("Ignore the error and retry polling job status.", e);
@@ -274,6 +287,11 @@ Job pollJob(
       return null;
     }
 
+    private static String formatBqStatusCommand(String projectId, String jobId) {
+      return String.format("bq show -j --format=prettyjson --project_id=%s %s",
+          projectId, jobId);
+    }
+
     @Override
     public JobStatistics dryRunQuery(String projectId, JobConfigurationQuery queryConfig)
         throws InterruptedException, IOException {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index 41e298c56fb9..49000d610924 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -141,6 +141,8 @@ private List<String> executeExtract(
           BigQueryHelpers.statusToPrettyString(extractJob.getStatus())));
     }
 
+    LOG.info("BigQuery extract job completed: {}", jobId);
+
     List<String> tempFiles = BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob);
     return ImmutableList.copyOf(tempFiles);
   }

From 17ad1efe7355b238efb5e341487a8e22660b3b77 Mon Sep 17 00:00:00 2001
From: Borisa Zivkovic <borisa.zivkovic@huawei.com>
Date: Wed, 3 May 2017 16:22:18 +0100
Subject: [PATCH 017/387] Use BinaryCombineLongFn in GroupIntoBatches

---
 .../beam/sdk/transforms/GroupIntoBatches.java | 51 +++++++------------
 1 file changed, 17 insertions(+), 34 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
index 02f3a8598f88..564baf4e8b3b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
@@ -23,7 +23,6 @@
 import com.google.common.collect.Iterables;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.Timer;
@@ -115,7 +114,7 @@ static class GroupIntoBatchesDoFn<K, InputT>
     private final StateSpec<BagState<InputT>> batchSpec;
 
     @StateId(NUM_ELEMENTS_IN_BATCH_ID)
-    private final StateSpec<CombiningState<Long, Long, Long>>
+    private final StateSpec<CombiningState<Long, long[], Long>>
         numElementsInBatchSpec;
 
     @StateId(KEY_ID)
@@ -131,35 +130,19 @@ static class GroupIntoBatchesDoFn<K, InputT>
       this.batchSize = batchSize;
       this.allowedLateness = allowedLateness;
       this.batchSpec = StateSpecs.bag(inputValueCoder);
-      this.numElementsInBatchSpec =
-          StateSpecs.combining(
-              VarLongCoder.of(),
-              new Combine.CombineFn<Long, Long, Long>() {
-
-                @Override
-                public Long createAccumulator() {
-                  return 0L;
-                }
-
-                @Override
-                public Long addInput(Long accumulator, Long input) {
-                  return accumulator + input;
-                }
-
-                @Override
-                public Long mergeAccumulators(Iterable<Long> accumulators) {
-                  long sum = 0L;
-                  for (Long accumulator : accumulators) {
-                    sum += accumulator;
-                  }
-                  return sum;
-                }
-
-                @Override
-                public Long extractOutput(Long accumulator) {
-                  return accumulator;
-                }
-              });
+      this.numElementsInBatchSpec = StateSpecs.combining(new Combine.BinaryCombineLongFn() {
+
+        @Override
+        public long identity() {
+          return 0L;
+        }
+
+        @Override
+        public long apply(long left, long right) {
+          return left + right;
+        }
+
+      });
 
       this.keySpec = StateSpecs.value(inputKeyCoder);
       // prefetch every 20% of batchSize elements. Do not prefetch if batchSize is too little
@@ -171,7 +154,7 @@ public void processElement(
         @TimerId(END_OF_WINDOW_ID) Timer timer,
         @StateId(BATCH_ID) BagState<InputT> batch,
         @StateId(NUM_ELEMENTS_IN_BATCH_ID)
-            CombiningState<Long, Long, Long> numElementsInBatch,
+            CombiningState<Long, long[], Long> numElementsInBatch,
         @StateId(KEY_ID) ValueState<K> key,
         ProcessContext c,
         BoundedWindow window) {
@@ -203,7 +186,7 @@ public void onTimerCallback(
         @StateId(KEY_ID) ValueState<K> key,
         @StateId(BATCH_ID) BagState<InputT> batch,
         @StateId(NUM_ELEMENTS_IN_BATCH_ID)
-            CombiningState<Long, Long, Long> numElementsInBatch,
+            CombiningState<Long, long[], Long> numElementsInBatch,
         BoundedWindow window) {
       LOGGER.debug(
           "*** END OF WINDOW *** for timer timestamp {} in windows {}",
@@ -215,7 +198,7 @@ private void flushBatch(
         Context c,
         ValueState<K> key,
         BagState<InputT> batch,
-        CombiningState<Long, Long, Long> numElementsInBatch) {
+        CombiningState<Long, long[], Long> numElementsInBatch) {
       Iterable<InputT> values = batch.read();
       // when the timer fires, batch state might be empty
       if (!Iterables.isEmpty(values)) {

From 70dad36f099ea0b454e2900302f7e7f866579f79 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 3 May 2017 13:50:46 -0700
Subject: [PATCH 018/387] [BEAM-2152] Remove gcloud auth as application default
 credentials does it

---
 sdks/python/apache_beam/internal/gcp/auth.py  | 32 -------------------
 .../runners/dataflow/internal/apiclient.py    |  3 +-
 2 files changed, 1 insertion(+), 34 deletions(-)

diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py
index 53c2d077c8aa..3c8dd64d9759 100644
--- a/sdks/python/apache_beam/internal/gcp/auth.py
+++ b/sdks/python/apache_beam/internal/gcp/auth.py
@@ -26,7 +26,6 @@
 from oauth2client.client import GoogleCredentials
 from oauth2client.client import OAuth2Credentials
 
-from apache_beam.utils import processes
 from apache_beam.utils import retry
 
 
@@ -94,27 +93,6 @@ def _refresh(self, http_request):
                          datetime.timedelta(seconds=token_data['expires_in']))
 
 
-class _GCloudWrapperCredentials(OAuth2Credentials):
-  """Credentials class wrapping gcloud credentials via shell."""
-
-  def __init__(self, user_agent, **kwds):
-    super(_GCloudWrapperCredentials, self).__init__(
-        None, None, None, None, None, None, user_agent, **kwds)
-
-  def _refresh(self, http_request):
-    """Gets an access token using the gcloud client."""
-    try:
-      gcloud_process = processes.Popen(
-          ['gcloud', 'auth', 'print-access-token'], stdout=processes.PIPE)
-    except OSError:
-      message = 'gcloud tool not found so falling back to using ' +\
-                'application default credentials'
-      logging.warning(message)
-      raise AuthenticationException(message)
-    output, _ = gcloud_process.communicate()
-    self.access_token = output.strip()
-
-
 def get_service_credentials():
   """Get credentials to access Google services."""
   user_agent = 'beam-python-sdk/1.0'
@@ -134,16 +112,6 @@ def get_service_credentials():
         'https://www.googleapis.com/auth/datastore'
     ]
 
-    try:
-      credentials = _GCloudWrapperCredentials(user_agent)
-      # Check if we are able to get an access token. If not fallback to
-      # application default credentials.
-      credentials.get_access_token()
-      return credentials
-    except AuthenticationException:
-      logging.warning('Unable to find credentials from gcloud.')
-
-    # Falling back to application default credentials.
     try:
       credentials = GoogleCredentials.get_application_default()
       credentials = credentials.create_scoped(client_scopes)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 26ed7b694d9a..e64cf8da3a13 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -411,8 +411,7 @@ def stage_file(self, gcs_or_local_path, file_name, stream,
       if e.status_code in reportable_errors:
         raise IOError(('Could not upload to GCS path %s: %s. Please verify '
                        'that credentials are valid and that you have write '
-                       'access to the specified path. Stale credentials can be '
-                       'refreshed by executing "gcloud auth login".') %
+                       'access to the specified path.') %
                       (gcs_or_local_path, reportable_errors[e.status_code]))
       raise
     logging.info('Completed GCS upload to %s', gcs_location)

From c102d277e22cef8001c0f78d3a5ed00000e8d99d Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 17:50:20 -0700
Subject: [PATCH 019/387] AvroIOTest: stop using IOChannelUtils, remove invalid
 test

---
 .../org/apache/beam/sdk/io/AvroIOTest.java    | 28 -------------------
 1 file changed, 28 deletions(-)

diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
index 1506aa9cbc6d..47b847fc153e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
@@ -57,11 +57,9 @@
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
-import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.testing.UsesTestStream;
 import org.apache.beam.sdk.testing.ValidatesRunner;
@@ -71,14 +69,11 @@
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -98,11 +93,6 @@ public class AvroIOTest {
   @Rule
   public TemporaryFolder tmpFolder = new TemporaryFolder();
 
-  @BeforeClass
-  public static void setupClass() {
-    IOChannelUtils.registerIOFactoriesAllowOverride(TestPipeline.testingPipelineOptions());
-  }
-
   @Test
   public void testAvroIOGetName() {
     assertEquals("AvroIO.Read", AvroIO.read(String.class).from("/tmp/foo*/baz").getName());
@@ -570,22 +560,4 @@ public void testWriteDisplayData() {
     assertThat(displayData, hasDisplayItem("numShards", 100));
     assertThat(displayData, hasDisplayItem("codec", CodecFactory.snappyCodec().toString()));
   }
-
-  @Test
-  @Category(ValidatesRunner.class)
-  @Ignore("[BEAM-436] DirectRunner ValidatesRunner tempLocation configuration insufficient")
-  public void testPrimitiveWriteDisplayData() throws IOException {
-    PipelineOptions options = DisplayDataEvaluator.getDefaultOptions();
-    String tempRoot = options.as(TestPipelineOptions.class).getTempRoot();
-    String outputPath = IOChannelUtils.getFactory(tempRoot).resolve(tempRoot, "foo");
-
-    DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(options);
-
-    AvroIO.Write<GenericRecord> write =
-        AvroIO.writeGenericRecords(Schema.create(Schema.Type.STRING)).to(outputPath);
-
-    Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(write);
-    assertThat("AvroIO.Write should include the file pattern in its primitive transform",
-        displayData, hasItem(hasDisplayItem("fileNamePattern")));
-  }
 }

From e5a38ed2610b8ef72192e5a1b9a5630578300164 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 17:55:32 -0700
Subject: [PATCH 020/387] DataflowRunner: switch from IOChannels to FileSystems
 for creating files

---
 .../apache/beam/runners/dataflow/DataflowRunner.java  | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 2b54ba74e7d3..1a806b9faecb 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -84,6 +84,7 @@
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.FileBasedSink;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.io.WriteFiles;
@@ -114,7 +115,6 @@
 import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.InstanceBuilder;
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.NameUtils;
@@ -192,9 +192,6 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
    * @return The newly created runner.
    */
   public static DataflowRunner fromOptions(PipelineOptions options) {
-    // (Re-)register standard IO factories. Clobbers any prior credentials.
-    IOChannelUtils.registerIOFactoriesAllowOverride(options);
-
     DataflowPipelineOptions dataflowOptions =
         PipelineOptionsValidator.validate(DataflowPipelineOptions.class, options);
     ArrayList<String> missing = new ArrayList<>();
@@ -578,9 +575,11 @@ public DataflowPipelineJob run(Pipeline pipeline) {
           fileLocation.startsWith("/") || fileLocation.startsWith("gs://"),
           "Location must be local or on Cloud Storage, got %s.",
           fileLocation);
+      ResourceId fileResource = FileSystems.matchNewResource(fileLocation, false /* isDirectory */);
       String workSpecJson = DataflowPipelineTranslator.jobToString(newJob);
-      try (PrintWriter printWriter = new PrintWriter(
-          Channels.newOutputStream(IOChannelUtils.create(fileLocation, MimeTypes.TEXT)))) {
+      try (PrintWriter printWriter =
+          new PrintWriter(
+              Channels.newOutputStream(FileSystems.create(fileResource, MimeTypes.TEXT)))) {
         printWriter.print(workSpecJson);
         LOG.info("Printed job specification to {}", fileLocation);
       } catch (IOException ex) {

From dc9e00485afaf737557bc6a82750e45ecba34926 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 18:01:29 -0700
Subject: [PATCH 021/387] DataflowPipelineOptions: use FileSystems, not
 IOChannelUtils, to resolve staging location

---
 .../options/DataflowPipelineOptions.java      | 14 +++++---------
 .../DataflowPipelineTranslatorTest.java       |  2 +-
 .../runners/dataflow/DataflowRunnerTest.java  |  4 +++-
 .../options/DataflowPipelineOptionsTest.java  | 19 +++++++++----------
 4 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 0796b6d5dd16..11618af9b1fe 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -17,11 +17,12 @@
  */
 package org.apache.beam.runners.dataflow.options;
 
-import java.io.IOException;
 import org.apache.beam.runners.dataflow.DataflowRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubOptions;
 import org.apache.beam.sdk.options.ApplicationNameOptions;
@@ -32,7 +33,6 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.options.Validation;
-import org.apache.beam.sdk.util.IOChannelUtils;
 
 /**
  * Options that can be used to configure the {@link DataflowRunner}.
@@ -137,13 +137,9 @@ public String create(PipelineOptions options) {
             "Error constructing default value for stagingLocation: gcpTempLocation is not"
             + " a valid GCS path, %s. ", gcpTempLocation), e);
       }
-      try {
-        return IOChannelUtils.resolve(gcpTempLocation, "staging");
-      } catch (IOException e) {
-        throw new IllegalArgumentException(String.format(
-            "Unable to resolve stagingLocation from gcpTempLocation: %s."
-            + " Please set the staging location explicitly.", gcpTempLocation), e);
-      }
+      return FileSystems.matchNewResource(gcpTempLocation, true /* isDirectory */)
+          .resolve("staging", StandardResolveOptions.RESOLVE_DIRECTORY)
+          .toString();
     }
   }
 }
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index a6ad8c5ca2fd..70f00fb0f2a0 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -224,7 +224,7 @@ public void testSettingOfSdkPipelineOptions() throws IOException {
     assertThat(optionsMap, hasEntry("jobName", (Object) "some-job-name"));
     assertThat(optionsMap, hasEntry("tempLocation", (Object) "gs://somebucket/some/path"));
     assertThat(optionsMap,
-        hasEntry("stagingLocation", (Object) "gs://somebucket/some/path/staging"));
+        hasEntry("stagingLocation", (Object) "gs://somebucket/some/path/staging/"));
     assertThat(optionsMap, hasEntry("stableUniqueNames", (Object) "WARNING"));
     assertThat(optionsMap, hasEntry("streaming", (Object) false));
     assertThat(optionsMap, hasEntry("numberOfWorkerHarnessThreads", (Object) 0));
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index c0dfbee6951e..5aebf29d1033 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -157,8 +157,9 @@ public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
       }
     });
     when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_STAGING_BUCKET))).thenReturn(true);
+    when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_STAGING_BUCKET))).thenReturn(true);
     when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_TEMP_BUCKET))).thenReturn(true);
-    when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_TEMP_BUCKET + "/staging"))).
+    when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_TEMP_BUCKET + "/staging/"))).
         thenReturn(true);
     when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_PROFILE_BUCKET))).thenReturn(true);
     when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(NON_EXISTENT_BUCKET))).thenReturn(false);
@@ -826,6 +827,7 @@ public void testProjectDescription() throws IOException {
   @Test
   public void testInvalidNumberOfWorkerHarnessThreads() throws IOException {
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
+    FileSystems.setDefaultConfigInWorkers(options);
     options.setRunner(DataflowRunner.class);
     options.setProject("foo-12345");
 
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
index 30eee0e9a8b0..8b8fd6c83c96 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
@@ -22,10 +22,10 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage;
 
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.ResetDateTimeProvider;
 import org.apache.beam.sdk.testing.RestoreSystemProperties;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.NoopPathValidator;
 import org.junit.Rule;
 import org.junit.Test;
@@ -127,7 +127,6 @@ public void testUtf8UserNameAndApplicationNameIsNormalized() {
   @Test
   public void testStagingLocation() {
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
-    IOChannelUtils.registerIOFactoriesAllowOverride(options);
     options.setPathValidatorClass(NoopPathValidator.class);
     options.setTempLocation("gs://temp_location");
     options.setStagingLocation("gs://staging_location");
@@ -138,21 +137,21 @@ public void testStagingLocation() {
   @Test
   public void testDefaultToTempLocation() {
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
-    IOChannelUtils.registerIOFactoriesAllowOverride(options);
+    FileSystems.setDefaultConfigInWorkers(options);
     options.setPathValidatorClass(NoopPathValidator.class);
-    options.setTempLocation("gs://temp_location");
-    assertEquals("gs://temp_location", options.getGcpTempLocation());
-    assertEquals("gs://temp_location/staging", options.getStagingLocation());
+    options.setTempLocation("gs://temp_location/");
+    assertEquals("gs://temp_location/", options.getGcpTempLocation());
+    assertEquals("gs://temp_location/staging/", options.getStagingLocation());
   }
 
   @Test
   public void testDefaultToGcpTempLocation() {
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
-    IOChannelUtils.registerIOFactoriesAllowOverride(options);
+    FileSystems.setDefaultConfigInWorkers(options);
     options.setPathValidatorClass(NoopPathValidator.class);
-    options.setTempLocation("gs://temp_location");
-    options.setGcpTempLocation("gs://gcp_temp_location");
-    assertEquals("gs://gcp_temp_location/staging", options.getStagingLocation());
+    options.setTempLocation("gs://temp_location/");
+    options.setGcpTempLocation("gs://gcp_temp_location/");
+    assertEquals("gs://gcp_temp_location/staging/", options.getStagingLocation());
   }
 
   @Test

From 5519fe51aea3e95fa0ea4a36c7b036917b7f94ec Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 18:02:42 -0700
Subject: [PATCH 022/387] Stop registering IOChannelFactories in SDK harness,
 runners, tests

---
 .../apex/translation/utils/SerializablePipelineOptions.java   | 2 --
 .../flink/translation/utils/SerializedPipelineOptions.java    | 2 --
 .../beam/runners/spark/translation/SparkRuntimeContext.java   | 4 +---
 .../main/java/org/apache/beam/sdk/runners/PipelineRunner.java | 4 +---
 .../main/java/org/apache/beam/sdk/testing/TestPipeline.java   | 2 --
 .../src/main/java/org/apache/beam/fn/harness/FnHarness.java   | 3 ---
 6 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
index 14476b57c26c..02afa7a31e46 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
@@ -27,7 +27,6 @@
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
 
 /**
@@ -63,7 +62,6 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept
         .as(ApexPipelineOptions.class);
 
     if (FILE_SYSTEMS_INTIIALIZED.compareAndSet(false, true)) {
-      IOChannelUtils.registerIOFactoriesAllowOverride(pipelineOptions);
       FileSystems.setDefaultConfigInWorkers(pipelineOptions);
     }
   }
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
index f717fd76c5e6..84f3bf40f59c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
@@ -27,7 +27,6 @@
 import java.io.Serializable;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
 
 /**
@@ -57,7 +56,6 @@ public PipelineOptions getPipelineOptions() {
       try {
         pipelineOptions = createMapper().readValue(serializedOptions, PipelineOptions.class);
 
-        IOChannelUtils.registerIOFactoriesAllowOverride(pipelineOptions);
         FileSystems.setDefaultConfigInWorkers(pipelineOptions);
       } catch (IOException e) {
         throw new RuntimeException("Couldn't deserialize the PipelineOptions.", e);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index 6bba863ae657..e006143cc0f3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -26,7 +26,6 @@
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.IOChannelUtils;
 
 /**
  * The SparkRuntimeContext allows us to define useful features on the client side before our
@@ -78,8 +77,7 @@ static PipelineOptions getOrInit(String serializedPipelineOptions) {
             pipelineOptions = deserializePipelineOptions(serializedPipelineOptions);
           }
         }
-        // register IO factories.
-        IOChannelUtils.registerIOFactoriesAllowOverride(pipelineOptions);
+        // Register standard FileSystems.
         FileSystems.setDefaultConfigInWorkers(pipelineOptions);
       }
       return pipelineOptions;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java
index a318dfca99bf..229e04ffa12d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java
@@ -24,7 +24,6 @@
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.InstanceBuilder;
 
 /**
@@ -44,8 +43,7 @@ public static PipelineRunner<? extends PipelineResult> fromOptions(PipelineOptio
     checkNotNull(options);
     PipelineOptionsValidator.validate(PipelineOptions.class, options);
 
-    // (Re-)register standard IO factories. Clobbers any prior credentials.
-    IOChannelUtils.registerIOFactoriesAllowOverride(options);
+    // (Re-)register standard FileSystems. Clobbers any prior credentials.
     FileSystems.setDefaultConfigInWorkers(options);
 
     @SuppressWarnings("unchecked")
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index 4d0cc2b36423..868dcbd56d55 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -46,7 +46,6 @@
 import org.apache.beam.sdk.options.PipelineOptions.CheckEnabled;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestRule;
@@ -400,7 +399,6 @@ public static PipelineOptions testingPipelineOptions() {
       }
       options.setStableUniqueNames(CheckEnabled.ERROR);
 
-      IOChannelUtils.registerIOFactoriesAllowOverride(options);
       FileSystems.setDefaultConfigInWorkers(options);
       return options;
     } catch (IOException e) {
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
index 24f826c5c522..05ab44f7ff5a 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java
@@ -33,7 +33,6 @@
 import org.apache.beam.fn.v1.BeamFnApi;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -90,8 +89,6 @@ public static void main(String[] args) throws Exception {
   public static void main(PipelineOptions options,
       BeamFnApi.ApiServiceDescriptor loggingApiServiceDescriptor,
       BeamFnApi.ApiServiceDescriptor controlApiServiceDescriptor) throws Exception {
-    IOChannelUtils.registerIOFactories(options);
-
     ManagedChannelFactory channelFactory = ManagedChannelFactory.from(options);
     StreamObserverFactory streamObserverFactory = StreamObserverFactory.fromOptions(options);
     PrintStream originalErrStream = System.err;

From f4e7c02d0a91f5df9ddcb2e868ef5adb86b3f85c Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 19:34:04 -0700
Subject: [PATCH 023/387] FileSystems: add match(String) for single spec

---
 .../org/apache/beam/sdk/io/FileSystems.java     | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index 0110a0cc269e..a3af8d9860e9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -19,6 +19,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Verify.verify;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
@@ -107,6 +108,22 @@ public static List<MatchResult> match(List<String> specs) throws IOException {
     return getFileSystemInternal(getOnlyScheme(specs)).match(specs);
   }
 
+
+  /**
+   * Like {@link #match(List)}, but for a single resource specification.
+   *
+   * <p>The function {@link #match(List)} is preferred when matching multiple patterns, as it allows
+   * for bulk API calls to remote filesystems.
+   */
+  public static MatchResult match(String spec) throws IOException {
+    List<MatchResult> matches = match(Collections.singletonList(spec));
+    verify(
+        matches.size() == 1,
+        "FileSystem implementation for %s did not return exactly one MatchResult: %s",
+        spec,
+        matches);
+    return matches.get(0);
+  }
   /**
    * Returns the {@link Metadata} for a single file resource. Expects a resource specification
    * {@code spec} that matches a single result.

From d050477065cf8cf27b1334ed7bebad5b30e5d8e4 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 18:11:34 -0700
Subject: [PATCH 024/387] WriteWithShardingFactoryTest: switch to FileSystems

---
 .../direct/WriteWithShardingFactoryTest.java  | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
index 18940d2d5bcc..f28c8cf3aede 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
@@ -32,7 +32,6 @@
 import java.io.Reader;
 import java.nio.CharBuffer;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
@@ -42,9 +41,11 @@
 import org.apache.beam.sdk.io.DefaultFilenamePolicy;
 import org.apache.beam.sdk.io.FileBasedSink;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.LocalResources;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -53,7 +54,6 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFnTester;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
@@ -72,7 +72,7 @@
  */
 @RunWith(JUnit4.class)
 public class WriteWithShardingFactoryTest {
-  public static final int INPUT_SIZE = 10000;
+  private static final int INPUT_SIZE = 10000;
   @Rule public TemporaryFolder tmp = new TemporaryFolder();
   private WriteWithShardingFactory<Object> factory = new WriteWithShardingFactory<>();
   @Rule public final TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
@@ -86,19 +86,22 @@ public void dynamicallyReshardedWrite() throws Exception {
     Collections.shuffle(strs);
 
     String fileName = "resharded_write";
-    String outputPath = tmp.getRoot().getAbsolutePath();
-    String targetLocation = IOChannelUtils.resolve(outputPath, fileName);
+    String targetLocation = tmp.getRoot().toPath().resolve(fileName).toString();
+    String targetLocationGlob = targetLocation + '*';
+
     // TextIO is implemented in terms of the WriteFiles PTransform. When sharding is not specified,
     // resharding should be automatically applied
     p.apply(Create.of(strs)).apply(TextIO.write().to(targetLocation));
-
     p.run();
 
-    Collection<String> files = IOChannelUtils.getFactory(outputPath).match(targetLocation + "*");
+    List<Metadata> matches = FileSystems.match(targetLocationGlob).metadata();
     List<String> actuals = new ArrayList<>(strs.size());
-    for (String file : files) {
-      CharBuffer buf = CharBuffer.allocate((int) new File(file).length());
-      try (Reader reader = new FileReader(file)) {
+    List<String> files = new ArrayList<>(strs.size());
+    for (Metadata match : matches) {
+      String filename = match.resourceId().toString();
+      files.add(filename);
+      CharBuffer buf = CharBuffer.allocate((int) new File(filename).length());
+      try (Reader reader = new FileReader(filename)) {
         reader.read(buf);
         buf.flip();
       }

From 56b1280d51b2458de93c7a5e2f31822fe6cf1718 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 18:18:52 -0700
Subject: [PATCH 025/387] ReadTranslator: remove outdated comment

---
 .../java/org/apache/beam/runners/dataflow/ReadTranslator.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
index c304c3260642..396c3051e5f8 100755
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
@@ -46,8 +46,6 @@ public static <T> void translateReadHelper(Source<T> source,
       PTransform<?, ? extends PValue> transform,
       TranslationContext context) {
     try {
-      // TODO: Move this validation out of translation once IOChannelUtils is portable
-      // and can be reconstructed on the worker.
       if (source instanceof FileBasedSource) {
         ValueProvider<String> filePatternOrSpec =
             ((FileBasedSource<?>) source).getFileOrPatternSpecProvider();

From 7e3d5723db6d1118df31e6af496b1aa2240607c9 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 3 May 2017 19:34:25 -0700
Subject: [PATCH 026/387] TFRecordIO: switch to FileSystems from IOChannelUtils

---
 .../src/main/java/org/apache/beam/sdk/io/TFRecordIO.java | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
index 3198829c00ef..6350fb57aa98 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
@@ -35,6 +35,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.Read.Bounded;
+import org.apache.beam.sdk.io.fs.MatchResult;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -43,7 +44,6 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -158,12 +158,11 @@ public PCollection<byte[]> expand(PBegin input) {
       if (getValidate()) {
         checkState(getFilepattern().isAccessible(), "Cannot validate with a RVP.");
         try {
+          MatchResult matches = FileSystems.match(getFilepattern().get());
           checkState(
-              !IOChannelUtils.getFactory(getFilepattern().get())
-                  .match(getFilepattern().get())
-                  .isEmpty(),
+              !matches.metadata().isEmpty(),
               "Unable to find any files matching %s",
-              getFilepattern());
+              getFilepattern().get());
         } catch (IOException e) {
           throw new IllegalStateException(
               String.format("Failed to validate %s", getFilepattern().get()), e);

From ab8f92fcc272e7eca70720fb91df584769376ecc Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 2 May 2017 17:53:57 -0700
Subject: [PATCH 027/387] Remove POutput#recordAsOutput

Add PValue#setDefaultName, for composite POutputs to use when they are
being finalized.

Expand POutput#finishSpecifyingOutput to take the name of the transform
that is being finished.
---
 .../beam/sdk/runners/TransformHierarchy.java  |  8 ++--
 .../apache/beam/sdk/values/PCollection.java   |  4 +-
 .../beam/sdk/values/PCollectionList.java      | 14 +++---
 .../beam/sdk/values/PCollectionTuple.java     | 17 ++++---
 .../org/apache/beam/sdk/values/PDone.java     | 12 ++---
 .../org/apache/beam/sdk/values/POutput.java   | 25 ++---------
 .../apache/beam/sdk/values/PValueBase.java    | 44 ++++++-------------
 .../sdk/runners/TransformHierarchyTest.java   |  3 +-
 .../apache/beam/sdk/transforms/ParDoTest.java |  4 +-
 .../beam/sdk/io/gcp/bigquery/WriteResult.java | 21 +--------
 10 files changed, 47 insertions(+), 105 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 92361942fc8a..703aeb684346 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -171,14 +171,14 @@ public void setOutput(POutput output) {
     for (PValue value : output.expand().values()) {
       if (!producers.containsKey(value)) {
         producers.put(value, current);
+        value.finishSpecifyingOutput(
+            current.getFullName(), unexpandedInputs.get(current), current.transform);
       }
-      value.finishSpecifyingOutput(unexpandedInputs.get(current), current.transform);
       producerInput.put(value, unexpandedInputs.get(current));
     }
-    output.finishSpecifyingOutput(unexpandedInputs.get(current), current.transform);
+    output.finishSpecifyingOutput(
+        current.getFullName(), unexpandedInputs.get(current), current.transform);
     current.setOutput(output);
-    // TODO: Replace with a "generateDefaultNames" method.
-    output.recordAsOutput(current.toAppliedPTransform());
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
index 20e5d68946b4..16223223749c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
@@ -83,9 +83,9 @@ public class PCollection<T> extends PValueBase implements PValue {
 
   @Override
   public void finishSpecifyingOutput(
-      PInput input, PTransform<?, ?> transform) {
+      String transformName, PInput input, PTransform<?, ?> transform) {
     this.coderOrFailure = inferCoderOrFail(input, transform, getPipeline().getCoderRegistry());
-    super.finishSpecifyingOutput(input, transform);
+    super.finishSpecifyingOutput(transformName, input, transform);
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java
index 7b45deb391a3..48c36495ef0d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java
@@ -24,7 +24,6 @@
 import java.util.Map;
 import java.util.Objects;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.Partition;
@@ -230,21 +229,20 @@ public Map<TupleTag<?>, PValue> expand() {
   }
 
   @Override
-  public void recordAsOutput(AppliedPTransform<?, ?, ?> transform) {
+  public void finishSpecifyingOutput(
+      String transformName, PInput input, PTransform<?, ?> transform) {
+    // All component PCollections will have already been finished.
     int i = 0;
     for (TaggedPValue tpv : pcollections) {
       @SuppressWarnings("unchecked")
       PCollection<T> pc = (PCollection<T>) tpv.getValue();
-      pc.recordAsOutput(transform, "out" + i);
+      if (pc.getName().equals(PValueBase.defaultName(transformName))) {
+        pc.setName(String.format("%s.%s%s", transformName, "out", i));
+      }
       i++;
     }
   }
 
-  @Override
-  public void finishSpecifyingOutput(PInput input, PTransform<?, ?> transform) {
-    // All component PCollections will have already been finished.
-  }
-
   @Override
   public boolean equals(Object other) {
     if (!(other instanceof PCollectionList)) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
index ce67e94764bc..5027df652ddf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
@@ -23,7 +23,6 @@
 import java.util.Map;
 import java.util.Objects;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.util.WindowingStrategy;
@@ -236,22 +235,22 @@ public Map<TupleTag<?>, PValue> expand() {
   }
 
   @Override
-  public void recordAsOutput(AppliedPTransform<?, ?, ?> transform) {
+  public void finishSpecifyingOutput(
+      String transformName, PInput input, PTransform<?, ?> transform) {
+    // All component PCollections will already have been finished. Update their names if
+    // appropriate.
     int i = 0;
     for (Map.Entry<TupleTag<?>, PCollection<?>> entry
-             : pcollectionMap.entrySet()) {
+        : pcollectionMap.entrySet()) {
       TupleTag<?> tag = entry.getKey();
       PCollection<?> pc = entry.getValue();
-      pc.recordAsOutput(transform, tag.getOutName(i));
+      if (pc.getName().equals(PValueBase.defaultName(transformName))) {
+        pc.setName(String.format("%s.%s", transformName, tag.getOutName(i)));
+      }
       i++;
     }
   }
 
-  @Override
-  public void finishSpecifyingOutput(PInput input, PTransform<?, ?> transform) {
-    // All component PCollections will already have been finished
-  }
-
   @Override
   public boolean equals(Object other) {
     if (!(other instanceof PCollectionTuple)) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java
index 5c9800de9d1d..92473b94e6a4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java
@@ -21,7 +21,6 @@
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.WriteFiles;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
@@ -56,13 +55,8 @@ public Map<TupleTag<?>, PValue> expand() {
     return Collections.emptyMap();
   }
 
-  /** Does nothing; there are no concrete outputs to record. */
+  /** Does nothing; there is nothing to finish specifying. */
   @Override
-  public void recordAsOutput(AppliedPTransform<?, ?, ?> transform) {}
-
-  /**
-   * Does nothing; there is nothing to finish specifying.
-   */
-  @Override
-  public void finishSpecifyingOutput(PInput input, PTransform<?, ?> transform) { }
+  public void finishSpecifyingOutput(
+      String transformName, PInput input, PTransform<?, ?> transform) {}
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/POutput.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/POutput.java
index bb01beb2d0c6..c6d15e327f0f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/POutput.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/POutput.java
@@ -19,7 +19,6 @@
 
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
@@ -47,19 +46,6 @@ public interface POutput {
    */
   Map<TupleTag<?>, PValue> expand();
 
-  /**
-   * Records that this {@code POutput} is an output of the given
-   * {@code PTransform}.
-   *
-   * <p>For a compound {@code POutput}, it is advised to call
-   * this method on each component {@code POutput}.
-   *
-   * <p>This is not intended to be invoked by user code, but
-   * is automatically invoked as part of applying the
-   * producing {@link PTransform}.
-   */
-  void recordAsOutput(AppliedPTransform<?, ?, ?> transform);
-
   /**
    * As part of applying the producing {@link PTransform}, finalizes this output to make it ready
    * for being used as an input and for running.
@@ -67,12 +53,9 @@ public interface POutput {
    * <p>This includes ensuring that all {@link PCollection PCollections} have {@link
    * org.apache.beam.sdk.coders.Coder Coders} specified or defaulted.
    *
-   * <p>Automatically invoked whenever this {@link POutput} is output, after {@link
-   * PValue#finishSpecifyingOutput(PInput, PTransform)} has been called on each component {@link
-   * PValue} returned by {@link #expand()}.
-   *
-   * @deprecated see BEAM-1199
+   * <p>Automatically invoked whenever this {@link POutput} is output, after
+   * {@link PValue#finishSpecifyingOutput(String, PInput, PTransform)} has been called on each
+   * component {@link PValue} returned by {@link #expand()}.
    */
-  @Deprecated
-  void finishSpecifyingOutput(PInput input, PTransform<?, ?> transform);
+  void finishSpecifyingOutput(String transformName, PInput input, PTransform<?, ?> transform);
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
index 7ab58088fb90..4de0589d1222 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
@@ -17,10 +17,11 @@
  */
 package org.apache.beam.sdk.values;
 
+import static com.google.common.base.Preconditions.checkState;
+
 import java.util.Collections;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.NameUtils;
 
@@ -65,10 +66,7 @@ public String getName() {
    * already been finalized and may no longer be set.
    */
   public PValueBase setName(String name) {
-    if (finishedSpecifying) {
-      throw new IllegalStateException(
-          "cannot change the name of " + this + " once it's been used");
-    }
+    checkState(!finishedSpecifying, "cannot change the name of %s once it's been used", this);
     this.name = name;
     return this;
   }
@@ -104,26 +102,6 @@ protected PValueBase() {
    */
   private boolean finishedSpecifying = false;
 
-  @Override
-  public void recordAsOutput(AppliedPTransform<?, ?, ?> transform) {
-    recordAsOutput(transform, "out");
-  }
-
-  /**
-   * Records that this {@link PValueBase} is an output with the
-   * given name of the given {@link AppliedPTransform} in the given
-   * {@link Pipeline}.
-   *
-   * <p>To be invoked only by {@link POutput#recordAsOutput}
-   * implementations.  Not to be invoked directly by user code.
-   */
-  protected void recordAsOutput(AppliedPTransform<?, ?, ?> transform,
-                                String outName) {
-    if (name == null) {
-      name = transform.getFullName() + "." + outName;
-    }
-  }
-
   /**
    * Returns whether this {@link PValueBase} has been finalized, and
    * its core properties, e.g., name, can no longer be changed.
@@ -165,11 +143,15 @@ public Pipeline getPipeline() {
     return pipeline;
   }
 
-  /**
-   * Default behavior for {@link #finishSpecifyingOutput(PInput, PTransform)}} is
-   * to do nothing. Override if your {@link PValue} requires
-   * finalization.
-   */
   @Override
-  public void finishSpecifyingOutput(PInput input, PTransform<?, ?> transform) { }
+  public void finishSpecifyingOutput(
+      String transformName, PInput input, PTransform<?, ?> transform) {
+    if (name == null) {
+      setName(defaultName(transformName));
+    }
+  }
+
+  static String defaultName(String transformName) {
+    return String.format("%s.%s", transformName, "out");
+  }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
index 1b884e2dd365..e495758c788e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
@@ -145,7 +145,8 @@ public void producingOwnAndOthersOutputsFails() {
     final PCollectionList<Long> appended =
         pcList.and(
             PCollection.<Long>createPrimitiveOutputInternal(
-                pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED));
+                    pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)
+                .setName("prim"));
     hierarchy.pushNode(
         "AddPc",
         pcList,
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 26904aa09d15..d4475c925266 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -1244,7 +1244,9 @@ public void testTaggedOutputUnregisteredExplicitCoder() throws Exception {
     outputTuple.get(additionalOutputTag).apply(View.<TestDummy>asSingleton());
 
     assertEquals(new TestDummyCoder(), outputTuple.get(additionalOutputTag).getCoder());
-    outputTuple.get(additionalOutputTag).finishSpecifyingOutput(input, pardo); // Check for crashes
+    outputTuple
+        .get(additionalOutputTag)
+        .finishSpecifyingOutput("ParDo", input, pardo); // Check for crashes
     assertEquals(new TestDummyCoder(),
         outputTuple.get(additionalOutputTag).getCoder()); // Check for corruption
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
index d137f052f554..bc18e8e032e9 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
@@ -20,7 +20,6 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
@@ -55,23 +54,7 @@ public Pipeline getPipeline() {
     return pipeline;
   }
 
-  /**
-   * Records that this {@link WriteResult} is an output with the given name of the given {@link
-   * AppliedPTransform}.
-   *
-   * <p>By default, does nothing.
-   *
-   * <p>To be invoked only by {@link POutput#recordAsOutput} implementations. Not to be invoked
-   * directly by user code.
-   */
-  @Override
-  public void recordAsOutput(AppliedPTransform<?, ?, ?> transform) {}
-
-  /**
-   * Default behavior for {@link #finishSpecifyingOutput(PInput, PTransform)}} is
-   * to do nothing. Override if your {@link PValue} requires
-   * finalization.
-   */
   @Override
-  public void finishSpecifyingOutput(PInput input, PTransform<?, ?> transform) { }
+  public void finishSpecifyingOutput(
+      String transformName, PInput input, PTransform<?, ?> transform) {}
 }

From c141ebc4ac8088c782d6b74ee38042a72ed342f3 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Wed, 3 May 2017 17:39:32 -0700
Subject: [PATCH 028/387] One more TaggedOutput name change.

---
 sdks/python/apache_beam/runners/common.pxd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/runners/common.pxd b/sdks/python/apache_beam/runners/common.pxd
index f3395c1ddf10..53f7aa8cc962 100644
--- a/sdks/python/apache_beam/runners/common.pxd
+++ b/sdks/python/apache_beam/runners/common.pxd
@@ -21,7 +21,7 @@ from apache_beam.utils.windowed_value cimport WindowedValue
 from apache_beam.metrics.execution cimport ScopedMetricsContainer
 
 
-cdef type OutputValue, TimestampedValue
+cdef type TaggedOutput, TimestampedValue
 
 
 cdef class Receiver(object):

From 4cdfe36ce3b423dfb0f26a371a255d5490e7b593 Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Tue, 2 May 2017 12:01:38 -0700
Subject: [PATCH 029/387] Update JavaDoc of checkpoint, and expose
 `enable.auto.commit` for more visible.

---
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java     | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index 8f94b8a488b0..8ab33d1c5b69 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -204,14 +204,23 @@
  *
  * <h3>Partition Assignment and Checkpointing</h3>
  * The Kafka partitions are evenly distributed among splits (workers).
- * Checkpointing is fully supported and each split can resume from previous checkpoint. See
- * {@link UnboundedKafkaSource#split(int, PipelineOptions)} for more details on
+ *
+ * <p>Checkpointing is fully supported and each split can resume from previous checkpoint
+ * (to the extent supported by runner).
+ * See {@link UnboundedKafkaSource#split(int, PipelineOptions)} for more details on
  * splits and checkpoint support.
  *
- * <p>When the pipeline starts for the first time without any checkpoint, the source starts
+ * <p>When the pipeline starts for the first time, or without any checkpoint, the source starts
  * consuming from the <em>latest</em> offsets. You can override this behavior to consume from the
  * beginning by setting appropriate appropriate properties in {@link ConsumerConfig}, through
  * {@link Read#updateConsumerProperties(Map)}.
+ * You can also enable offset auto_commit in Kafka to resume from last committed.
+ *
+ * <p>In summary, KafkaIO.read follows below sequence to set initial offset:<br>
+ * 1. {@link KafkaCheckpointMark} provided by runner;<br>
+ * 2. Consumer offset stored in Kafka when
+ * {@code ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG = true};<br>
+ * 3. Start from <em>latest</em> offset by default;
  *
  * <h3>Writing to Kafka</h3>
  *

From 48c8ed17623b3f36b7aeebb2e2ca585a259d2fec Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 4 May 2017 09:08:28 -0700
Subject: [PATCH 030/387] Use CloudObject encodings in the Beam Fn Harness

This mostly reverts commit e2b5d6ea8a5d41ac27245a4999f59a73dfe24c43.
---
 sdks/java/harness/pom.xml                     | 10 ------
 .../runners/core/BeamFnDataReadRunner.java    | 21 +++++++++---
 .../runners/core/BeamFnDataWriteRunner.java   | 22 ++++++++++---
 .../control/ProcessBundleHandlerTest.java     | 32 +++++++++----------
 .../core/BeamFnDataReadRunnerTest.java        | 16 ++++++----
 .../core/BeamFnDataWriteRunnerTest.java       | 16 ++++++----
 6 files changed, 68 insertions(+), 49 deletions(-)

diff --git a/sdks/java/harness/pom.xml b/sdks/java/harness/pom.xml
index 73f08cc012fb..d00dfe4b14ce 100644
--- a/sdks/java/harness/pom.xml
+++ b/sdks/java/harness/pom.xml
@@ -63,16 +63,6 @@
       <artifactId>beam-sdks-java-core</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-runners-core-construction-java</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-common-runner-api</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-sdks-java-core</artifactId>
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
index 7c4a5e8fa272..e6928d1aa43b 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataReadRunner.java
@@ -18,8 +18,10 @@
 
 package org.apache.beam.runners.core;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
+import com.google.protobuf.BytesValue;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Map;
@@ -28,9 +30,9 @@
 import org.apache.beam.fn.harness.data.BeamFnDataClient;
 import org.apache.beam.fn.harness.fn.ThrowingConsumer;
 import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.core.construction.Coders;
+import org.apache.beam.runners.dataflow.util.CloudObject;
+import org.apache.beam.runners.dataflow.util.CloudObjects;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.MessageWithComponents;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.slf4j.Logger;
@@ -47,6 +49,8 @@
 public class BeamFnDataReadRunner<OutputT> {
   private static final Logger LOG = LoggerFactory.getLogger(BeamFnDataReadRunner.class);
 
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
   private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
   private final Collection<ThrowingConsumer<WindowedValue<OutputT>>> consumers;
   private final Supplier<String> processBundleInstructionIdSupplier;
@@ -71,12 +75,19 @@ public BeamFnDataReadRunner(
     this.beamFnDataClientFactory = beamFnDataClientFactory;
     this.consumers = ImmutableList.copyOf(FluentIterable.concat(outputMap.values()));
 
-    MessageWithComponents runnerApiCoder =
-        coderSpec.getFunctionSpec().getData().unpack(MessageWithComponents.class);
     @SuppressWarnings("unchecked")
     Coder<WindowedValue<OutputT>> coder =
         (Coder<WindowedValue<OutputT>>)
-            Coders.fromProto(runnerApiCoder.getCoder(), runnerApiCoder.getComponents());
+            CloudObjects.coderFromCloudObject(
+                CloudObject.fromSpec(
+                    OBJECT_MAPPER.readValue(
+                        coderSpec
+                            .getFunctionSpec()
+                            .getData()
+                            .unpack(BytesValue.class)
+                            .getValue()
+                            .newInput(),
+                        Map.class)));
     this.coder = coder;
   }
 
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
index 3a11def393fc..a78da5d201d7 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/runners/core/BeamFnDataWriteRunner.java
@@ -18,14 +18,17 @@
 
 package org.apache.beam.runners.core;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.protobuf.BytesValue;
 import java.io.IOException;
+import java.util.Map;
 import java.util.function.Supplier;
 import org.apache.beam.fn.harness.data.BeamFnDataClient;
 import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
 import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.core.construction.Coders;
+import org.apache.beam.runners.dataflow.util.CloudObject;
+import org.apache.beam.runners.dataflow.util.CloudObjects;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.MessageWithComponents;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 
@@ -37,6 +40,8 @@
  * For each request, call {@link #registerForOutput()} to start and call {@link #close()} to finish.
  */
 public class BeamFnDataWriteRunner<InputT> {
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
   private final BeamFnApi.ApiServiceDescriptor apiServiceDescriptor;
   private final BeamFnApi.Target outputTarget;
   private final Coder<WindowedValue<InputT>> coder;
@@ -58,12 +63,19 @@ public BeamFnDataWriteRunner(
     this.processBundleInstructionIdSupplier = processBundleInstructionIdSupplier;
     this.outputTarget = outputTarget;
 
-    MessageWithComponents runnerApiCoder =
-        coderSpec.getFunctionSpec().getData().unpack(MessageWithComponents.class);
     @SuppressWarnings("unchecked")
     Coder<WindowedValue<InputT>> coder =
         (Coder<WindowedValue<InputT>>)
-            Coders.fromProto(runnerApiCoder.getCoder(), runnerApiCoder.getComponents());
+            CloudObjects.coderFromCloudObject(
+                CloudObject.fromSpec(
+                    OBJECT_MAPPER.readValue(
+                        coderSpec
+                            .getFunctionSpec()
+                            .getData()
+                            .unpack(BytesValue.class)
+                            .getValue()
+                            .newInput(),
+                        Map.class)));
     this.coder = coder;
   }
 
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 654f98988cc0..1cdd087cfd47 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -33,6 +33,7 @@
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
@@ -61,7 +62,7 @@
 import org.apache.beam.fn.harness.fn.ThrowingConsumer;
 import org.apache.beam.fn.harness.fn.ThrowingRunnable;
 import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.core.construction.Coders;
+import org.apache.beam.runners.dataflow.util.CloudObjects;
 import org.apache.beam.runners.dataflow.util.DoFnInfo;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -91,6 +92,8 @@
 /** Tests for {@link ProcessBundleHandler}. */
 @RunWith(JUnit4.class)
 public class ProcessBundleHandlerTest {
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
   private static final Coder<WindowedValue<String>> STRING_CODER =
       WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
   private static final String LONG_CODER_SPEC_ID = "998L";
@@ -105,23 +108,18 @@ public class ProcessBundleHandlerTest {
   static {
     try {
       STRING_CODER_SPEC =
-          BeamFnApi.Coder.newBuilder()
-              .setFunctionSpec(
-                  BeamFnApi.FunctionSpec.newBuilder()
-                      .setId(STRING_CODER_SPEC_ID)
-                      .setData(Any.pack(Coders.toProto(STRING_CODER))))
-              .build();
+          BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
+          .setId(STRING_CODER_SPEC_ID)
+          .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
+              OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(STRING_CODER)))).build())))
+          .build();
       LONG_CODER_SPEC =
-          BeamFnApi.Coder.newBuilder()
-              .setFunctionSpec(
-                  BeamFnApi.FunctionSpec.newBuilder()
-                      .setId(STRING_CODER_SPEC_ID)
-                      .setData(
-                          Any.pack(
-                              Coders.toProto(
-                                  WindowedValue.getFullCoder(
-                                      VarLongCoder.of(), GlobalWindow.Coder.INSTANCE)))))
-              .build();
+          BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
+          .setId(STRING_CODER_SPEC_ID)
+          .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
+              OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(WindowedValue.getFullCoder(
+                  VarLongCoder.of(), GlobalWindow.Coder.INSTANCE))))).build())))
+          .build();
     } catch (IOException e) {
       throw new ExceptionInInitializerError(e);
     }
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
index 04a3615dde58..a3d4a1b0f109 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataReadRunnerTest.java
@@ -27,10 +27,13 @@
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Uninterruptibles;
 import com.google.protobuf.Any;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.BytesValue;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -45,7 +48,7 @@
 import org.apache.beam.fn.harness.test.TestExecutors;
 import org.apache.beam.fn.harness.test.TestExecutors.TestExecutorService;
 import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.core.construction.Coders;
+import org.apache.beam.runners.dataflow.util.CloudObjects;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -64,6 +67,8 @@
 /** Tests for {@link BeamFnDataReadRunner}. */
 @RunWith(JUnit4.class)
 public class BeamFnDataReadRunnerTest {
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
   private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
       .setApiServiceDescriptor(BeamFnApi.ApiServiceDescriptor.getDefaultInstance()).build();
   private static final BeamFnApi.FunctionSpec FUNCTION_SPEC = BeamFnApi.FunctionSpec.newBuilder()
@@ -73,11 +78,10 @@ public class BeamFnDataReadRunnerTest {
   private static final BeamFnApi.Coder CODER_SPEC;
   static {
     try {
-      CODER_SPEC =
-          BeamFnApi.Coder.newBuilder()
-              .setFunctionSpec(
-                  BeamFnApi.FunctionSpec.newBuilder().setData(Any.pack(Coders.toProto(CODER))))
-              .build();
+      CODER_SPEC = BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
+          .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
+              OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(CODER)))).build())))
+          .build();
     } catch (IOException e) {
       throw new ExceptionInInitializerError(e);
     }
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
index 9e50cd0e6212..338396650b41 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/runners/core/BeamFnDataWriteRunnerTest.java
@@ -28,14 +28,17 @@
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.protobuf.Any;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.BytesValue;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.beam.fn.harness.data.BeamFnDataClient;
 import org.apache.beam.fn.harness.fn.CloseableThrowingConsumer;
 import org.apache.beam.fn.v1.BeamFnApi;
-import org.apache.beam.runners.core.construction.Coders;
+import org.apache.beam.runners.dataflow.util.CloudObjects;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -52,6 +55,8 @@
 /** Tests for {@link BeamFnDataWriteRunner}. */
 @RunWith(JUnit4.class)
 public class BeamFnDataWriteRunnerTest {
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
   private static final BeamFnApi.RemoteGrpcPort PORT_SPEC = BeamFnApi.RemoteGrpcPort.newBuilder()
       .setApiServiceDescriptor(BeamFnApi.ApiServiceDescriptor.getDefaultInstance()).build();
   private static final BeamFnApi.FunctionSpec FUNCTION_SPEC = BeamFnApi.FunctionSpec.newBuilder()
@@ -61,11 +66,10 @@ public class BeamFnDataWriteRunnerTest {
   private static final BeamFnApi.Coder CODER_SPEC;
   static {
     try {
-      CODER_SPEC =
-          BeamFnApi.Coder.newBuilder()
-              .setFunctionSpec(
-                  BeamFnApi.FunctionSpec.newBuilder().setData(Any.pack(Coders.toProto(CODER))))
-              .build();
+      CODER_SPEC = BeamFnApi.Coder.newBuilder().setFunctionSpec(BeamFnApi.FunctionSpec.newBuilder()
+      .setData(Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(
+          OBJECT_MAPPER.writeValueAsBytes(CloudObjects.asCloudObject(CODER)))).build())))
+      .build();
     } catch (IOException e) {
       throw new ExceptionInInitializerError(e);
     }

From 95ade45eced4787eb67a9d4d13dae48ffb176919 Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Tue, 2 May 2017 19:00:29 +0300
Subject: [PATCH 031/387] [BEAM-1763] Verify PAssert execution in runners which
 support metrics.

---
 .../beam/runners/flink/FlinkRunner.java       |  3 ++
 .../beam/runners/spark/TestSparkRunner.java   | 47 -------------------
 .../ResumeFromCheckpointStreamingTest.java    | 12 +++--
 .../beam/sdk/metrics/MetricsEnvironment.java  |  5 ++
 .../apache/beam/sdk/testing/TestPipeline.java | 46 ++++++++++++++++--
 5 files changed, 57 insertions(+), 56 deletions(-)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
index 181ffda7571c..a5972efa1a15 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
@@ -31,6 +31,7 @@
 import java.util.TreeSet;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.PipelineRunner;
@@ -103,6 +104,8 @@ private FlinkRunner(FlinkPipelineOptions options) {
   public PipelineResult run(Pipeline pipeline) {
     logWarningIfPCollectionViewHasNonDeterministicKeyCoder(pipeline);
 
+    MetricsEnvironment.setMetricsSupported(true);
+
     LOG.info("Executing pipeline using FlinkRunner.");
 
     FlinkPipelineExecutionEnvironment env = new FlinkPipelineExecutionEnvironment(options);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 10e98b825803..1e678131d92b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -40,15 +40,11 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
-import org.apache.beam.sdk.metrics.MetricNameFilter;
-import org.apache.beam.sdk.metrics.MetricResult;
-import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -116,8 +112,6 @@ public SparkPipelineResult run(Pipeline pipeline) {
     }
     SparkPipelineResult result = null;
 
-    int expectedNumberOfAssertions = PAssert.countAsserts(pipeline);
-
     // clear state of Aggregators, Metrics and Watermarks if exists.
     AggregatorsAccumulator.clear();
     MetricsAccumulator.clear();
@@ -137,47 +131,6 @@ public SparkPipelineResult run(Pipeline pipeline) {
             String.format("Finish state %s is not allowed.", finishState),
             finishState,
             isOneOf(PipelineResult.State.STOPPED, PipelineResult.State.DONE));
-
-        // validate assertion succeeded (at least once).
-        long successAssertions = 0;
-        Iterable<MetricResult<Long>> counterResults = result.metrics().queryMetrics(
-            MetricsFilter.builder()
-                .addNameFilter(MetricNameFilter.named(PAssert.class, PAssert.SUCCESS_COUNTER))
-                .build()).counters();
-        for (MetricResult<Long> counter : counterResults) {
-          if (counter.attempted().longValue() > 0) {
-            successAssertions++;
-          }
-        }
-        Integer expectedAssertions = testSparkPipelineOptions.getExpectedAssertions() != null
-            ? testSparkPipelineOptions.getExpectedAssertions() : expectedNumberOfAssertions;
-        assertThat(
-            String.format(
-                "Expected %d successful assertions, but found %d.",
-                expectedAssertions, successAssertions),
-            successAssertions,
-            is(expectedAssertions.longValue()));
-        // validate assertion didn't fail.
-        long failedAssertions = 0;
-        Iterable<MetricResult<Long>> failCounterResults = result.metrics().queryMetrics(
-            MetricsFilter.builder()
-                .addNameFilter(MetricNameFilter.named(PAssert.class, PAssert.FAILURE_COUNTER))
-                .build()).counters();
-        for (MetricResult<Long> counter : failCounterResults) {
-          if (counter.attempted().longValue() > 0) {
-            failedAssertions++;
-          }
-        }
-        assertThat(
-            String.format("Found %d failed assertions.", failedAssertions),
-            failedAssertions,
-            is(0L));
-
-        LOG.info(
-            String.format(
-                "Successfully asserted pipeline %s with %d successful assertions.",
-                testSparkPipelineOptions.getJobName(),
-                successAssertions));
       } finally {
         try {
           // cleanup checkpoint dir.
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
index 7d7fd0880382..33571f0f1691 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
@@ -180,7 +180,8 @@ public void testWithResume() throws Exception {
     long successAssertions = 0;
     Iterable<MetricResult<Long>> counterResults = res.metrics().queryMetrics(
         MetricsFilter.builder()
-            .addNameFilter(MetricNameFilter.named(PAssert.class, PAssert.SUCCESS_COUNTER))
+            .addNameFilter(
+                MetricNameFilter.named(PAssertWithoutFlatten.class, PAssert.SUCCESS_COUNTER))
             .build()).counters();
     for (MetricResult<Long> counter : counterResults) {
       if (counter.attempted().longValue() > 0) {
@@ -196,7 +197,8 @@ public void testWithResume() throws Exception {
     long failedAssertions = 0;
     Iterable<MetricResult<Long>> failCounterResults = res.metrics().queryMetrics(
         MetricsFilter.builder()
-            .addNameFilter(MetricNameFilter.named(PAssert.class, PAssert.FAILURE_COUNTER))
+            .addNameFilter(MetricNameFilter.named(
+                PAssertWithoutFlatten.class, PAssert.FAILURE_COUNTER))
             .build()).counters();
     for (MetricResult<Long> counter : failCounterResults) {
       if (counter.attempted().longValue() > 0) {
@@ -330,8 +332,10 @@ public PDone expand(PCollection<Iterable<T>> input) {
     }
 
     private static class AssertDoFn<T> extends DoFn<Iterable<T>, Void> {
-      private final Counter success = Metrics.counter(PAssert.class, PAssert.SUCCESS_COUNTER);
-      private final Counter failure = Metrics.counter(PAssert.class, PAssert.FAILURE_COUNTER);
+      private final Counter success =
+          Metrics.counter(PAssertWithoutFlatten.class, PAssert.SUCCESS_COUNTER);
+      private final Counter failure =
+          Metrics.counter(PAssertWithoutFlatten.class, PAssert.FAILURE_COUNTER);
       private final T[] expected;
 
       AssertDoFn(T[] expected) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java
index 2942578b1e95..a4b311ffd43e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java
@@ -70,6 +70,11 @@ public static void setMetricsSupported(boolean supported) {
     METRICS_SUPPORTED.set(supported);
   }
 
+  /** Indicates whether metrics reporting is supported. */
+  public static boolean isMetricsSupported() {
+    return METRICS_SUPPORTED.get();
+  }
+
   /**
    * Set the {@link MetricsContainer} for the current thread.
    *
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index 868dcbd56d55..d8fe51d92c72 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -18,6 +18,8 @@
 package org.apache.beam.sdk.testing;
 
 import static com.google.common.base.Preconditions.checkState;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.TreeNode;
@@ -41,6 +43,10 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.MetricResult;
+import org.apache.beam.sdk.metrics.MetricsEnvironment;
+import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.apache.beam.sdk.options.ApplicationNameOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptions.CheckEnabled;
@@ -186,8 +192,8 @@ private void verifyPipelineExecution() {
           if (pipelineRunSucceeded() && !visitedAll(pipelineNodes)) {
             final boolean hasDanglingPAssert =
                 FluentIterable.from(pipelineNodes)
-                              .filter(Predicates.not(Predicates.in(runVisitedNodes)))
-                              .anyMatch(isPAssertNode);
+                    .filter(Predicates.not(Predicates.in(runVisitedNodes)))
+                    .anyMatch(isPAssertNode);
             if (hasDanglingPAssert) {
               throw new AbandonedNodeException("The pipeline contains abandoned PAssert(s).");
             } else {
@@ -319,12 +325,13 @@ public PipelineResult run() {
     checkState(
         enforcement.isPresent(),
         "Is your TestPipeline declaration missing a @Rule annotation? Usage: "
-        + "@Rule public final transient TestPipeline pipeline = TestPipeline.create();");
+            + "@Rule public final transient TestPipeline pipeline = TestPipeline.create();");
 
     final PipelineResult pipelineResult;
     try {
       enforcement.get().beforePipelineExecution();
       pipelineResult = super.run();
+      verifyPAssertsSucceeded(pipelineResult);
     } catch (RuntimeException exc) {
       Throwable cause = exc.getCause();
       if (cause instanceof AssertionError) {
@@ -385,8 +392,8 @@ public static PipelineOptions testingPipelineOptions() {
           Strings.isNullOrEmpty(beamTestPipelineOptions)
               ? PipelineOptionsFactory.create()
               : PipelineOptionsFactory.fromArgs(
-                      MAPPER.readValue(beamTestPipelineOptions, String[].class))
-                  .as(TestPipelineOptions.class);
+              MAPPER.readValue(beamTestPipelineOptions, String[].class))
+              .as(TestPipelineOptions.class);
 
       options.as(ApplicationNameOptions.class).setAppName(getAppName());
       // If no options were specified, set some reasonable defaults
@@ -488,6 +495,35 @@ private static Optional<StackTraceElement> findCallersStackTrace() {
     return firstInstanceAfterTestPipeline;
   }
 
+  /**
+   * Verifies all {{@link PAssert PAsserts}} in the pipeline have been executed and were successful.
+   *
+   * <p>Note this only runs for runners which support Metrics. Runners which do not should verify
+   * this in some other way. See: https://issues.apache.org/jira/browse/BEAM-2001</p>
+   */
+  private void verifyPAssertsSucceeded(PipelineResult pipelineResult) {
+    if (MetricsEnvironment.isMetricsSupported()) {
+      long expectedNumberOfAssertions = (long) PAssert.countAsserts(this);
+
+      long successfulAssertions = 0;
+      Iterable<MetricResult<Long>> successCounterResults =
+          pipelineResult.metrics().queryMetrics(
+              MetricsFilter.builder()
+                  .addNameFilter(MetricNameFilter.named(PAssert.class, PAssert.SUCCESS_COUNTER))
+                  .build())
+              .counters();
+      for (MetricResult<Long> counter : successCounterResults) {
+        if (counter.attempted() > 0) {
+          successfulAssertions++;
+        }
+      }
+
+      assertThat(String
+          .format("Expected %d successful assertions, but found %d.", expectedNumberOfAssertions,
+              successfulAssertions), successfulAssertions, is(expectedNumberOfAssertions));
+    }
+  }
+
   private static class IsEmptyVisitor extends PipelineVisitor.Defaults {
     private boolean empty = true;
 

From 5555040d935c67f5cd48f2ffe2721a07fe6e0a50 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sat, 18 Mar 2017 12:16:06 +0100
Subject: [PATCH 032/387] [BEAM-1726] Fix empty side inputs in Flink Streaming
 Runner

---
 .../beam/runners/core/SideInputHandler.java   | 10 +++----
 .../wrappers/streaming/DoFnOperator.java      | 27 ++++++++++++++++++-
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
index 5c67148ea0e6..b29f9d0281bb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -161,11 +162,6 @@ public void addSideInputValue(
   @Override
   public <T> T get(PCollectionView<T> sideInput, BoundedWindow window) {
 
-    if (!isReady(sideInput, window)) {
-      throw new IllegalStateException(
-          "Side input " + sideInput + " is not ready for window " + window);
-    }
-
     @SuppressWarnings("unchecked")
     Coder<BoundedWindow> windowCoder =
         (Coder<BoundedWindow>) sideInput
@@ -181,6 +177,10 @@ public <T> T get(PCollectionView<T> sideInput, BoundedWindow window) {
 
     Iterable<WindowedValue<?>> elements = state.read();
 
+    if (elements == null) {
+      elements = Collections.emptyList();
+    }
+
     return sideInput.getViewFn().apply(elements);
   }
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index c62403637fc6..16bf5d253419 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -463,7 +463,32 @@ public void processWatermark1(Watermark mark) throws Exception {
 
   @Override
   public void processWatermark2(Watermark mark) throws Exception {
-    // ignore watermarks from the side-input input
+    if (mark.getTimestamp() == BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
+      // this means we will never see any more side input
+      pushbackDoFnRunner.startBundle();
+
+      BagState<WindowedValue<InputT>> pushedBack =
+          pushbackStateInternals.state(StateNamespaces.global(), pushedBackTag);
+
+      Iterable<WindowedValue<InputT>> pushedBackContents = pushedBack.read();
+      if (pushedBackContents != null) {
+        for (WindowedValue<InputT> elem : pushedBackContents) {
+
+          // we need to set the correct key in case the operator is
+          // a (keyed) window operator
+          setKeyContextElement1(new StreamRecord<>(elem));
+
+          doFnRunner.processElement(elem);
+        }
+      }
+
+      setPushedBackWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis());
+
+      pushbackDoFnRunner.finishBundle();
+
+      // maybe output a new watermark
+      processWatermark1(new Watermark(currentInputWatermark));
+    }
   }
 
   @Override

From 7c44935e1c47cce2ecfe842e37c2cf89f48d8583 Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Sat, 18 Mar 2017 15:21:45 +0200
Subject: [PATCH 033/387] [BEAM-1726] Fix RuntimeException throwing in
 FlinkStateInternals

---
 .../wrappers/streaming/state/FlinkStateInternals.java           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
index c033be672d9a..cea6e0f719fb 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
@@ -592,6 +592,8 @@ public void add(InputT value) {
         }
         current = combineFn.addInput(current, value);
         state.update(current);
+      } catch (RuntimeException re) {
+        throw re;
       } catch (Exception e) {
         throw new RuntimeException("Error adding to state." , e);
       }

From 0e2bb1808350cbebf771d0971deb06787732800d Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 07:49:08 +0100
Subject: [PATCH 034/387] [BEAM-1726] Fix Flatten with input copies in Flink
 Streaming Runner

---
 .../FlinkStreamingTransformTranslators.java   | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index c024493f2885..7339c01d9b81 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -966,10 +966,36 @@ public void flatMap(
 
       } else {
         DataStream<T> result = null;
+
+        // Determine DataStreams that we use as input several times. For those, we need to uniquify
+        // input streams because Flink seems to swallow watermarks when we have a union of one and
+        // the same stream.
+        Map<DataStream<T>, Integer> duplicates = new HashMap<>();
+        for (PValue input : allInputs.values()) {
+          DataStream<T> current = context.getInputDataStream(input);
+          Integer oldValue = duplicates.put(current, 1);
+          if (oldValue != null) {
+            duplicates.put(current, oldValue + 1);
+          }
+        }
+
         for (PValue input : allInputs.values()) {
           DataStream<T> current = context.getInputDataStream(input);
+
+          final Integer timesRequired = duplicates.get(current);
+          if (timesRequired > 1) {
+            current = current.flatMap(new FlatMapFunction<T, T>() {
+              private static final long serialVersionUID = 1L;
+
+              @Override
+              public void flatMap(T t, Collector<T> collector) throws Exception {
+                collector.collect(t);
+              }
+            });
+          }
           result = (result == null) ? current : result.union(current);
         }
+
         context.setOutputDataStream(context.getOutput(transform), result);
       }
     }

From 8d91a97b77fbda74c577d2cdbd507395834e147c Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Wed, 3 May 2017 21:06:00 +0300
Subject: [PATCH 035/387] [BEAM-1763] Replace usage of PipelineRule with
 TestPipeline in Spark runner tests

---
 runners/spark/pom.xml                         |  47 +++++++-
 .../runners/spark/SparkRunnerRegistrar.java   |   4 +-
 .../apache/beam/runners/spark/CacheTest.java  |  12 +-
 .../runners/spark/ForceStreamingTest.java     |  18 +--
 .../beam/runners/spark/PipelineRule.java      | 109 ------------------
 .../spark/ProvidedSparkContextTest.java       |  10 +-
 .../spark/SparkRunnerDebuggerTest.java        |  15 +--
 .../spark/SparkRunnerRegistrarTest.java       |   2 +-
 .../beam/runners/spark/StreamingTest.java     |  23 ++++
 .../metrics/sink/SparkMetricsSinkTest.java    |  12 +-
 .../runners/spark/io/AvroPipelineTest.java    |  10 +-
 .../beam/runners/spark/io/NumShardsTest.java  |   6 +-
 .../spark/translation/StorageLevelTest.java   |  31 ++++-
 .../streaming/CreateStreamTest.java           |  53 ++++-----
 .../ResumeFromCheckpointStreamingTest.java    |  50 +++++---
 .../streaming/StreamingSourceMetricsTest.java |  14 +--
 .../java/org/apache/beam/sdk/Pipeline.java    |   2 +-
 .../apache/beam/sdk/testing/TestPipeline.java |  21 +++-
 18 files changed, 217 insertions(+), 222 deletions(-)
 delete mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
 create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/StreamingTest.java

diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 38d250e7e733..f7200d6ed0cf 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -333,9 +333,6 @@
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
           <configuration>
-            <excludedGroups>
-              org.apache.beam.runners.spark.UsesCheckpointRecovery
-            </excludedGroups>
             <forkCount>1</forkCount>
             <reuseForks>false</reuseForks>
             <systemPropertyVariables>
@@ -344,6 +341,50 @@
               <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
             </systemPropertyVariables>
           </configuration>
+          <executions>
+            <execution>
+              <id>default-test</id>
+              <goals>
+                <goal>test</goal>
+              </goals>
+              <configuration>
+                <excludedGroups>
+                  org.apache.beam.runners.spark.UsesCheckpointRecovery,
+                  org.apache.beam.runners.spark.StreamingTest
+                </excludedGroups>
+                <systemPropertyVariables>
+                  <beamTestPipelineOptions>
+                    [
+                    "--runner=TestSparkRunner",
+                    "--streaming=false",
+                    "--enableSparkMetricSinks=true"
+                    ]
+                  </beamTestPipelineOptions>
+                </systemPropertyVariables>
+              </configuration>
+            </execution>
+            <execution>
+              <id>streaming-tests</id>
+              <phase>test</phase>
+              <goals>
+                <goal>test</goal>
+              </goals>
+              <configuration>
+                <groups>
+                  org.apache.beam.runners.spark.StreamingTest
+                </groups>
+                <systemPropertyVariables>
+                  <beamTestPipelineOptions>
+                    [
+                    "--runner=TestSparkRunner",
+                    "--forceStreaming=true",
+                    "--enableSparkMetricSinks=true"
+                    ]
+                  </beamTestPipelineOptions>
+                </systemPropertyVariables>
+              </configuration>
+            </execution>
+          </executions>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
index bedfda419f41..bf926dc54ecc 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
@@ -54,7 +54,9 @@ public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
   public static class Options implements PipelineOptionsRegistrar {
     @Override
     public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
-      return ImmutableList.<Class<? extends PipelineOptions>>of(SparkPipelineOptions.class);
+      return ImmutableList.<Class<? extends PipelineOptions>>of(
+          SparkPipelineOptions.class,
+          TestSparkPipelineOptions.class);
     }
   }
 }
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java
index c3b48d82fb22..24b2e7b3a358 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java
@@ -23,11 +23,11 @@
 import org.apache.beam.runners.spark.translation.SparkContextFactory;
 import org.apache.beam.runners.spark.translation.TransformTranslator;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.junit.Rule;
 import org.junit.Test;
 
 /**
@@ -36,12 +36,12 @@
  */
 public class CacheTest {
 
-  @Rule
-  public final transient PipelineRule pipelineRule = PipelineRule.batch();
-
   @Test
   public void cacheCandidatesUpdaterTest() throws Exception {
-    Pipeline pipeline = pipelineRule.createPipeline();
+    SparkPipelineOptions options =
+        PipelineOptionsFactory.create().as(TestSparkPipelineOptions.class);
+    options.setRunner(TestSparkRunner.class);
+    Pipeline pipeline = Pipeline.create(options);
     PCollection<String> pCollection = pipeline.apply(Create.of("foo", "bar"));
     // first read
     pCollection.apply(Count.<String>globally());
@@ -50,7 +50,7 @@ public void cacheCandidatesUpdaterTest() throws Exception {
     // will cache the RDD representing this PCollection
     pCollection.apply(Count.<String>globally());
 
-    JavaSparkContext jsc = SparkContextFactory.getSparkContext(pipelineRule.getOptions());
+    JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
     EvaluationContext ctxt = new EvaluationContext(jsc, pipeline);
     SparkRunner.CacheVisitor cacheVisitor =
         new SparkRunner.CacheVisitor(new TransformTranslator.Translator(), ctxt);
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
index b60faf2e238d..7bfc980304a3 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
@@ -25,9 +25,9 @@
 import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.junit.Rule;
 import org.junit.Test;
 
 
@@ -44,19 +44,23 @@
  */
 public class ForceStreamingTest {
 
-  @Rule
-  public final PipelineRule pipelineRule = PipelineRule.streaming();
-
   @Test
   public void test() throws IOException {
-    Pipeline pipeline = pipelineRule.createPipeline();
+    TestSparkPipelineOptions options =
+        PipelineOptionsFactory.create().as(TestSparkPipelineOptions.class);
+    options.setRunner(TestSparkRunner.class);
+    options.setForceStreaming(true);
+
+    // pipeline with a bounded read.
+    Pipeline pipeline = Pipeline.create(options);
 
     // apply the BoundedReadFromUnboundedSource.
     BoundedReadFromUnboundedSource<?> boundedRead =
         Read.from(CountingSource.unbounded()).withMaxNumRecords(-1);
-    //noinspection unchecked
     pipeline.apply(boundedRead);
-    TestSparkRunner runner = TestSparkRunner.fromOptions(pipelineRule.getOptions());
+
+    // adapt reads
+    TestSparkRunner runner = TestSparkRunner.fromOptions(options);
     runner.adaptBoundedReads(pipeline);
 
     UnboundedReadDetector unboundedReadDetector = new UnboundedReadDetector();
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
deleted file mode 100644
index f8499f301e92..000000000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark;
-
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.joda.time.Duration;
-import org.junit.rules.ExternalResource;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TemporaryFolder;
-import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-
-/**
- * A {@link org.junit.Rule} to provide a {@link Pipeline} instance for Spark runner tests.
- */
-public class PipelineRule implements TestRule {
-
-  private final SparkPipelineRule delegate;
-  private final RuleChain chain;
-
-  private PipelineRule(SparkPipelineRule delegate) {
-    TestName testName = new TestName();
-    this.delegate = delegate;
-    this.delegate.setTestName(testName);
-    this.chain = RuleChain.outerRule(testName).around(this.delegate);
-  }
-
-  public static PipelineRule streaming() {
-    return new PipelineRule(new SparkStreamingPipelineRule());
-  }
-
-  public static PipelineRule batch() {
-    return new PipelineRule(new SparkPipelineRule());
-  }
-
-  public Duration batchDuration() {
-    return Duration.millis(delegate.options.getBatchIntervalMillis());
-  }
-
-  public TestSparkPipelineOptions getOptions() {
-    return delegate.options;
-  }
-
-  public Pipeline createPipeline() {
-    return Pipeline.create(delegate.options);
-  }
-
-  @Override
-  public Statement apply(Statement statement, Description description) {
-    return chain.apply(statement, description);
-  }
-
-  private static class SparkStreamingPipelineRule extends SparkPipelineRule {
-
-    private final TemporaryFolder temporaryFolder = new TemporaryFolder();
-
-    @Override
-    protected void before() throws Throwable {
-      super.before();
-      temporaryFolder.create();
-      options.setForceStreaming(true);
-      options.setCheckpointDir(
-          temporaryFolder.newFolder(options.getJobName()).toURI().toURL().toString());
-    }
-
-    @Override
-    protected void after() {
-      temporaryFolder.delete();
-    }
-  }
-
-  private static class SparkPipelineRule extends ExternalResource {
-
-    protected final TestSparkPipelineOptions options =
-        PipelineOptionsFactory.as(TestSparkPipelineOptions.class);
-
-    private TestName testName;
-
-    public void setTestName(TestName testName) {
-      this.testName = testName;
-    }
-
-    @Override
-    protected void before() throws Throwable {
-      options.setRunner(TestSparkRunner.class);
-      options.setEnableSparkMetricSinks(false);
-      options.setJobName(
-          testName != null ? testName.getMethodName() : "test-at-" + System.currentTimeMillis());
-    }
-  }
-}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java
index 36ba863e4272..811299308f26 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java
@@ -27,9 +27,11 @@
 import java.util.Set;
 import org.apache.beam.runners.spark.examples.WordCount;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.values.PCollection;
@@ -95,7 +97,9 @@ private void testWithValidProvidedContext(JavaSparkContext jsc) throws Exception
         PAssert.that(output).containsInAnyOrder(EXPECTED_COUNT_SET);
 
         // Run test from pipeline
-        p.run().waitUntilFinish();
+        PipelineResult result = p.run();
+
+        TestPipeline.verifyPAssertsSucceeded(p, result);
     }
 
     private void testWithInvalidContext(JavaSparkContext jsc) {
@@ -104,11 +108,9 @@ private void testWithInvalidContext(JavaSparkContext jsc) {
         Pipeline p = Pipeline.create(options);
         PCollection<String> inputWords = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder
                 .of()));
-        PCollection<String> output = inputWords.apply(new WordCount.CountWords())
+        inputWords.apply(new WordCount.CountWords())
                 .apply(MapElements.via(new WordCount.FormatAsTextFn()));
 
-        PAssert.that(output).containsInAnyOrder(EXPECTED_COUNT_SET);
-
         try {
             p.run().waitUntilFinish();
             fail("Should throw an exception when The provided Spark context is null or stopped");
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java
index ea058b2b749c..90097511c45b 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java
@@ -27,6 +27,8 @@
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.io.kafka.KafkaIO;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Distinct;
@@ -48,7 +50,6 @@
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
-import org.junit.Rule;
 import org.junit.Test;
 
 
@@ -57,15 +58,9 @@
  */
 public class SparkRunnerDebuggerTest {
 
-  @Rule
-  public final PipelineRule batchPipelineRule = PipelineRule.batch();
-
-  @Rule
-  public final PipelineRule streamingPipelineRule = PipelineRule.streaming();
-
   @Test
   public void debugBatchPipeline() {
-    TestSparkPipelineOptions options = batchPipelineRule.getOptions();
+    PipelineOptions options = PipelineOptionsFactory.create().as(TestSparkPipelineOptions.class);
     options.setRunner(SparkRunnerDebugger.class);
 
     Pipeline pipeline = Pipeline.create(options);
@@ -111,7 +106,9 @@ public void debugBatchPipeline() {
 
   @Test
   public void debugStreamingPipeline() {
-    TestSparkPipelineOptions options = streamingPipelineRule.getOptions();
+    TestSparkPipelineOptions options =
+        PipelineOptionsFactory.create().as(TestSparkPipelineOptions.class);
+    options.setForceStreaming(true);
     options.setRunner(SparkRunnerDebugger.class);
 
     Pipeline pipeline = Pipeline.create(options);
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
index 4e1fd7c5afee..75899f9a395f 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
@@ -38,7 +38,7 @@ public class SparkRunnerRegistrarTest {
   @Test
   public void testOptions() {
     assertEquals(
-        ImmutableList.of(SparkPipelineOptions.class),
+        ImmutableList.of(SparkPipelineOptions.class, TestSparkPipelineOptions.class),
         new SparkRunnerRegistrar.Options().getPipelineOptions());
   }
 
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/StreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/StreamingTest.java
new file mode 100644
index 000000000000..a34c184d3b66
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/StreamingTest.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark;
+
+/**
+ * Category tag for tests that should be run in streaming mode.
+ */
+public interface StreamingTest {}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java
index b0ad97253d74..fff95cbcaf11 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java
@@ -26,11 +26,10 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
-import org.apache.beam.runners.spark.PipelineRule;
 import org.apache.beam.runners.spark.examples.WordCount;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.values.PCollection;
@@ -48,12 +47,7 @@ public class SparkMetricsSinkTest {
   public ExternalResource inMemoryMetricsSink = new InMemoryMetricsSinkRule();
 
   @Rule
-  public final PipelineRule pipelineRule = PipelineRule.batch();
-
-  private Pipeline createSparkPipeline() {
-    pipelineRule.getOptions().setEnableSparkMetricSinks(true);
-    return pipelineRule.createPipeline();
-  }
+  public final TestPipeline pipeline = TestPipeline.create();
 
   private void runPipeline() {
     final List<String> words =
@@ -62,8 +56,6 @@ private void runPipeline() {
     final Set<String> expectedCounts =
         ImmutableSet.of("hi: 5", "there: 1", "sue: 2", "bob: 2");
 
-    final Pipeline pipeline = createSparkPipeline();
-
     final PCollection<String> output =
         pipeline
         .apply(Create.of(words).withCoder(StringUtf8Coder.of()))
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java
index 7188dc598fbd..adde8d258c8b 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java
@@ -33,9 +33,8 @@
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericDatumWriter;
 import org.apache.avro.generic.GenericRecord;
-import org.apache.beam.runners.spark.PipelineRule;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.AvroIO;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.values.PCollection;
 import org.junit.Before;
 import org.junit.Rule;
@@ -54,7 +53,7 @@ public class AvroPipelineTest {
   public final TemporaryFolder tmpDir = new TemporaryFolder();
 
   @Rule
-  public final PipelineRule pipelineRule = PipelineRule.batch();
+  public final TestPipeline pipeline = TestPipeline.create();
 
   @Before
   public void setUp() throws IOException {
@@ -72,11 +71,10 @@ public void testGeneric() throws Exception {
     savedRecord.put("siblingnames", Lists.newArrayList("Jimmy", "Jane"));
     populateGenericFile(Lists.newArrayList(savedRecord), schema);
 
-    Pipeline p = pipelineRule.createPipeline();
-    PCollection<GenericRecord> input = p.apply(
+    PCollection<GenericRecord> input = pipeline.apply(
         AvroIO.readGenericRecords(schema).from(inputFile.getAbsolutePath()));
     input.apply(AvroIO.writeGenericRecords(schema).to(outputDir.getAbsolutePath()));
-    p.run().waitUntilFinish();
+    pipeline.run();
 
     List<GenericRecord> records = readGenericFile();
     assertEquals(Lists.newArrayList(savedRecord), records);
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
index 50217447b0af..55ee93827541 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
@@ -30,11 +30,10 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
-import org.apache.beam.runners.spark.PipelineRule;
 import org.apache.beam.runners.spark.examples.WordCount;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.values.PCollection;
@@ -59,7 +58,7 @@ public class NumShardsTest {
   public final TemporaryFolder tmpDir = new TemporaryFolder();
 
   @Rule
-  public final PipelineRule pipelineRule = PipelineRule.batch();
+  public final TestPipeline p = TestPipeline.create();
 
   @Before
   public void setUp() throws IOException {
@@ -69,7 +68,6 @@ public void setUp() throws IOException {
 
   @Test
   public void testText() throws Exception {
-    Pipeline p = pipelineRule.createPipeline();
     PCollection<String> inputWords = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of()));
     PCollection<String> output = inputWords.apply(new WordCount.CountWords())
         .apply(MapElements.via(new WordCount.FormatAsTextFn()));
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
index 2b7b87b11f09..8f2e681c9e91 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
@@ -15,30 +15,49 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.beam.runners.spark.translation;
 
-import org.apache.beam.runners.spark.PipelineRule;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.values.PCollection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 
+
 /**
  * Test the RDD storage level defined by user.
  */
 public class StorageLevelTest {
 
+  private static String beamTestPipelineOptions;
+
   @Rule
-  public final transient PipelineRule pipelineRule = PipelineRule.batch();
+  public final TestPipeline pipeline = TestPipeline.create();
+
+  @BeforeClass
+  public static void init() {
+    beamTestPipelineOptions =
+        System.getProperty(TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS);
+
+    System.setProperty(
+        TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS,
+        beamTestPipelineOptions.replace("]", ", \"--storageLevel=DISK_ONLY\"]"));
+  }
+
+  @AfterClass
+  public static void teardown() {
+    System.setProperty(
+        TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS,
+        beamTestPipelineOptions);
+  }
 
   @Test
   public void test() throws Exception {
-    pipelineRule.getOptions().setStorageLevel("DISK_ONLY");
-    Pipeline pipeline = pipelineRule.createPipeline();
-
     PCollection<String> pCollection = pipeline.apply(Create.of("foo"));
 
     // by default, the Spark runner doesn't cache the RDD if it accessed only one time.
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
index dd52c05985eb..770e0c054a92 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
@@ -24,13 +24,14 @@
 
 import java.io.IOException;
 import java.io.Serializable;
-import org.apache.beam.runners.spark.PipelineRule;
 import org.apache.beam.runners.spark.ReuseSparkContextRule;
+import org.apache.beam.runners.spark.SparkPipelineOptions;
+import org.apache.beam.runners.spark.StreamingTest;
 import org.apache.beam.runners.spark.io.CreateStream;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -61,6 +62,7 @@
 import org.joda.time.Instant;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
 
 
@@ -74,10 +76,11 @@
  * {@link org.apache.spark.streaming.dstream.QueueInputDStream} and advance the system's WMs.
  * //TODO: add synchronized/processing time trigger.
  */
+@Category(StreamingTest.class)
 public class CreateStreamTest implements Serializable {
 
   @Rule
-  public final transient PipelineRule pipelineRule = PipelineRule.streaming();
+  public final transient TestPipeline p = TestPipeline.create();
   @Rule
   public final transient ReuseSparkContextRule noContextResue = ReuseSparkContextRule.no();
   @Rule
@@ -85,10 +88,9 @@ public class CreateStreamTest implements Serializable {
 
   @Test
   public void testLateDataAccumulating() throws IOException {
-    Pipeline p = pipelineRule.createPipeline();
     Instant instant = new Instant(0);
     CreateStream<Integer> source =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .emptyBatch()
             .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(6)))
             .nextBatch(
@@ -159,9 +161,8 @@ public Void apply(Iterable<Integer> input) {
 
   @Test
   public void testDiscardingMode() throws IOException {
-    Pipeline p = pipelineRule.createPipeline();
     CreateStream<String> source =
-        CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
+        CreateStream.of(StringUtf8Coder.of(), batchDuration())
             .nextBatch(
                 TimestampedValue.of("firstPane", new Instant(100)),
                 TimestampedValue.of("alsoFirstPane", new Instant(200)))
@@ -208,10 +209,9 @@ public void testDiscardingMode() throws IOException {
 
   @Test
   public void testFirstElementLate() throws IOException {
-    Pipeline p = pipelineRule.createPipeline();
     Instant lateElementTimestamp = new Instant(-1_000_000);
     CreateStream<String> source =
-        CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
+        CreateStream.of(StringUtf8Coder.of(), batchDuration())
             .emptyBatch()
             .advanceWatermarkForNextBatch(new Instant(0))
             .nextBatch(
@@ -242,10 +242,9 @@ public void testFirstElementLate() throws IOException {
 
   @Test
   public void testElementsAtAlmostPositiveInfinity() throws IOException {
-    Pipeline p = pipelineRule.createPipeline();
     Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
     CreateStream<String> source =
-        CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
+        CreateStream.of(StringUtf8Coder.of(), batchDuration())
             .nextBatch(
                 TimestampedValue.of("foo", endOfGlobalWindow),
                 TimestampedValue.of("bar", endOfGlobalWindow))
@@ -267,13 +266,12 @@ public void testElementsAtAlmostPositiveInfinity() throws IOException {
 
   @Test
   public void testMultipleStreams() throws IOException {
-    Pipeline p = pipelineRule.createPipeline();
     CreateStream<String> source =
-        CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
+        CreateStream.of(StringUtf8Coder.of(), batchDuration())
             .nextBatch("foo", "bar")
             .advanceNextBatchWatermarkToInfinity();
     CreateStream<Integer> other =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .nextBatch(1, 2, 3, 4)
             .advanceNextBatchWatermarkToInfinity();
 
@@ -298,10 +296,9 @@ public void testMultipleStreams() throws IOException {
 
   @Test
   public void testFlattenedWithWatermarkHold() throws IOException {
-    Pipeline p = pipelineRule.createPipeline();
     Instant instant = new Instant(0);
     CreateStream<Integer> source1 =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .emptyBatch()
             .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(5)))
             .nextBatch(
@@ -310,7 +307,7 @@ public void testFlattenedWithWatermarkHold() throws IOException {
                 TimestampedValue.of(3, instant))
             .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(10)));
     CreateStream<Integer> source2 =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .emptyBatch()
             .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(1)))
             .nextBatch(
@@ -323,14 +320,14 @@ public void testFlattenedWithWatermarkHold() throws IOException {
             .advanceNextBatchWatermarkToInfinity();
 
     PCollection<Integer> windowed1 = p
-        .apply(source1)
-        .apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
+        .apply("CreateStream1", source1)
+        .apply("Window1", Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
             .triggering(AfterWatermark.pastEndOfWindow())
             .accumulatingFiredPanes()
             .withAllowedLateness(Duration.ZERO));
     PCollection<Integer> windowed2 = p
-        .apply(source2)
-        .apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
+        .apply("CreateStream2", source2)
+        .apply("Window2", Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
             .triggering(AfterWatermark.pastEndOfWindow())
             .accumulatingFiredPanes()
             .withAllowedLateness(Duration.ZERO));
@@ -357,10 +354,9 @@ public void testFlattenedWithWatermarkHold() throws IOException {
    */
   @Test
   public void testMultiOutputParDo() throws IOException {
-    Pipeline p = pipelineRule.createPipeline();
     Instant instant = new Instant(0);
     CreateStream<Integer> source1 =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .emptyBatch()
             .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(5)))
             .nextBatch(
@@ -397,7 +393,7 @@ public void process(ProcessContext context) {
   @Test
   public void testElementAtPositiveInfinityThrows() {
     CreateStream<Integer> source =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .nextBatch(TimestampedValue.of(-1, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L)))
             .advanceNextBatchWatermarkToInfinity();
     thrown.expect(IllegalArgumentException.class);
@@ -407,7 +403,7 @@ public void testElementAtPositiveInfinityThrows() {
   @Test
   public void testAdvanceWatermarkNonMonotonicThrows() {
     CreateStream<Integer> source =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .advanceWatermarkForNextBatch(new Instant(0L));
     thrown.expect(IllegalArgumentException.class);
     source
@@ -418,9 +414,14 @@ public void testAdvanceWatermarkNonMonotonicThrows() {
   @Test
   public void testAdvanceWatermarkEqualToPositiveInfinityThrows() {
     CreateStream<Integer> source =
-        CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+        CreateStream.of(VarIntCoder.of(), batchDuration())
             .advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L));
     thrown.expect(IllegalArgumentException.class);
     source.advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE);
   }
+
+  private Duration batchDuration() {
+    return Duration.millis(
+        (p.getOptions().as(SparkPipelineOptions.class)).getBatchIntervalMillis());
+  }
 }
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
index 33571f0f1691..584edace250a 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
@@ -28,15 +28,16 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Uninterruptibles;
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
-import org.apache.beam.runners.spark.PipelineRule;
 import org.apache.beam.runners.spark.ReuseSparkContextRule;
 import org.apache.beam.runners.spark.SparkPipelineResult;
 import org.apache.beam.runners.spark.TestSparkPipelineOptions;
+import org.apache.beam.runners.spark.TestSparkRunner;
 import org.apache.beam.runners.spark.UsesCheckpointRecovery;
 import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
 import org.apache.beam.runners.spark.io.MicrobatchSource;
@@ -53,6 +54,7 @@
 import org.apache.beam.sdk.metrics.MetricResult;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.metrics.MetricsFilter;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -81,11 +83,12 @@
 import org.joda.time.Instant;
 import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-
+import org.junit.rules.TemporaryFolder;
 
 /**
  * Tests DStream recovery from checkpoint.
@@ -96,24 +99,34 @@
  * {@link Metrics} values that are expected to resume from previous count and a side-input that is
  * expected to recover as well.
  */
-public class ResumeFromCheckpointStreamingTest {
+public class ResumeFromCheckpointStreamingTest implements Serializable {
   private static final EmbeddedKafkaCluster.EmbeddedZookeeper EMBEDDED_ZOOKEEPER =
       new EmbeddedKafkaCluster.EmbeddedZookeeper();
   private static final EmbeddedKafkaCluster EMBEDDED_KAFKA_CLUSTER =
       new EmbeddedKafkaCluster(EMBEDDED_ZOOKEEPER.getConnection(), new Properties());
   private static final String TOPIC = "kafka_beam_test_topic";
 
+  private transient TemporaryFolder temporaryFolder;
+
   @Rule
   public final transient ReuseSparkContextRule noContextReuse = ReuseSparkContextRule.no();
-  @Rule
-  public final transient PipelineRule pipelineRule = PipelineRule.streaming();
 
   @BeforeClass
-  public static void init() throws IOException {
+  public static void setup() throws IOException {
     EMBEDDED_ZOOKEEPER.startup();
     EMBEDDED_KAFKA_CLUSTER.startup();
   }
 
+  @Before
+  public void init() {
+    temporaryFolder = new TemporaryFolder();
+    try {
+      temporaryFolder.create();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   private static void produce(Map<String, Instant> messages) {
     Properties producerProps = new Properties();
     producerProps.putAll(EMBEDDED_KAFKA_CLUSTER.getProps());
@@ -148,7 +161,7 @@ public void testWithResume() throws Exception {
             .build();
 
     // first run should expect EOT matching the last injected element.
-    SparkPipelineResult res = run(pipelineRule, Optional.of(new Instant(400)), 0);
+    SparkPipelineResult res = run(Optional.of(new Instant(400)), 0);
 
     assertThat(res.metrics().queryMetrics(metricsFilter).counters(),
         hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(),
@@ -169,7 +182,7 @@ public void testWithResume() throws Exception {
     ));
 
     // recovery should resume from last read offset, and read the second batch of input.
-    res = runAgain(pipelineRule, 1);
+    res = runAgain(1);
     // assertions 2:
     assertThat(res.metrics().queryMetrics(metricsFilter).counters(),
         hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(),
@@ -209,18 +222,18 @@ public void testWithResume() throws Exception {
         String.format("Found %d failed assertions.", failedAssertions),
         failedAssertions,
         is(0L));
-
   }
 
-  private SparkPipelineResult runAgain(PipelineRule pipelineRule, int expectedAssertions) {
+  private SparkPipelineResult runAgain(int expectedAssertions) {
     // sleep before next run.
     Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
-    return run(pipelineRule, Optional.<Instant>absent(), expectedAssertions);
+    return run(Optional.<Instant>absent(), expectedAssertions);
   }
 
   @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
-  private static SparkPipelineResult run(
-      PipelineRule pipelineRule, Optional<Instant> stopWatermarkOption, int expectedAssertions) {
+  private SparkPipelineResult run(
+      Optional<Instant> stopWatermarkOption,
+      int expectedAssertions) {
     KafkaIO.Read<String, Instant> read = KafkaIO.<String, Instant>read()
         .withBootstrapServers(EMBEDDED_KAFKA_CLUSTER.getBrokerList())
         .withTopics(Collections.singletonList(TOPIC))
@@ -242,15 +255,21 @@ public Instant apply(KV<String, Instant> kv) {
           }
         });
 
-    TestSparkPipelineOptions options = pipelineRule.getOptions();
+    TestSparkPipelineOptions options =
+        PipelineOptionsFactory.create().as(TestSparkPipelineOptions.class);
     options.setSparkMaster("local[*]");
     options.setCheckpointDurationMillis(options.getBatchIntervalMillis());
     options.setExpectedAssertions(expectedAssertions);
+    options.setRunner(TestSparkRunner.class);
+    options.setEnableSparkMetricSinks(false);
+    options.setForceStreaming(true);
+    options.setCheckpointDir(temporaryFolder.getRoot().getPath());
     // timeout is per execution so it can be injected by the caller.
     if (stopWatermarkOption.isPresent()) {
       options.setStopPipelineWatermark(stopWatermarkOption.get().getMillis());
     }
-    Pipeline p = pipelineRule.createPipeline();
+
+    Pipeline p = Pipeline.create(options);
 
     PCollection<String> expectedCol =
         p.apply(Create.of(ImmutableList.of("side1", "side2")).withCoder(StringUtf8Coder.of()));
@@ -354,5 +373,4 @@ public void processElement(ProcessContext c) throws Exception {
       }
     }
   }
-
 }
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java
index 5a4b1b5e2919..df6027cf349b 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java
@@ -23,9 +23,7 @@
 import static org.junit.Assert.assertThat;
 
 import java.io.Serializable;
-import org.apache.beam.runners.spark.PipelineRule;
-import org.apache.beam.runners.spark.TestSparkPipelineOptions;
-import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.runners.spark.StreamingTest;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.io.Source;
@@ -34,10 +32,11 @@
 import org.apache.beam.sdk.metrics.MetricQueryResults;
 import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.apache.beam.sdk.metrics.SourceMetrics;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.joda.time.Duration;
 import org.junit.Rule;
 import org.junit.Test;
-
+import org.junit.experimental.categories.Category;
 
 /**
  * Verify metrics support for {@link Source Sources} in streaming pipelines.
@@ -47,14 +46,11 @@ public class StreamingSourceMetricsTest implements Serializable {
 
   // Force streaming pipeline using pipeline rule.
   @Rule
-  public final transient PipelineRule pipelineRule = PipelineRule.streaming();
+  public final transient TestPipeline pipeline = TestPipeline.create();
 
   @Test
+  @Category(StreamingTest.class)
   public void testUnboundedSourceMetrics() {
-    TestSparkPipelineOptions options = pipelineRule.getOptions();
-
-    Pipeline pipeline = Pipeline.create(options);
-
     final long numElements = 1000;
 
     pipeline.apply(
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index 6b15f0d3f655..6fa7a5a3b4e5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -460,7 +460,7 @@ OutputT applyTransform(String name, InputT input,
   private Set<String> usedFullNames = new HashSet<>();
   private CoderRegistry coderRegistry;
   private final List<String> unstableNames = new ArrayList<>();
-  private final PipelineOptions defaultOptions;
+  protected final PipelineOptions defaultOptions;
 
   protected Pipeline(PipelineOptions options) {
     this.defaultOptions = options;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index d8fe51d92c72..2d34b2249467 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -26,6 +26,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
@@ -244,8 +245,11 @@ public static class PipelineRunMissingException extends RuntimeException {
     }
   }
 
-  static final String PROPERTY_BEAM_TEST_PIPELINE_OPTIONS = "beamTestPipelineOptions";
+  /** System property used to set {@link TestPipelineOptions}. */
+  public static final String PROPERTY_BEAM_TEST_PIPELINE_OPTIONS = "beamTestPipelineOptions";
+
   static final String PROPERTY_USE_DEFAULT_DUMMY_RUNNER = "beamUseDummyRunner";
+
   private static final ObjectMapper MAPPER = new ObjectMapper().registerModules(
       ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
 
@@ -331,7 +335,7 @@ public PipelineResult run() {
     try {
       enforcement.get().beforePipelineExecution();
       pipelineResult = super.run();
-      verifyPAssertsSucceeded(pipelineResult);
+      verifyPAssertsSucceeded(this, pipelineResult);
     } catch (RuntimeException exc) {
       Throwable cause = exc.getCause();
       if (cause instanceof AssertionError) {
@@ -377,6 +381,15 @@ public TestPipeline enableAutoRunIfMissing(final boolean enable) {
     return this;
   }
 
+  @VisibleForTesting
+  @Override
+  /**
+   * Get this pipeline's options.
+   */
+  public PipelineOptions getOptions() {
+    return defaultOptions;
+  }
+
   @Override
   public String toString() {
     return "TestPipeline#" + getOptions().as(ApplicationNameOptions.class).getAppName();
@@ -501,9 +514,9 @@ private static Optional<StackTraceElement> findCallersStackTrace() {
    * <p>Note this only runs for runners which support Metrics. Runners which do not should verify
    * this in some other way. See: https://issues.apache.org/jira/browse/BEAM-2001</p>
    */
-  private void verifyPAssertsSucceeded(PipelineResult pipelineResult) {
+  public static void verifyPAssertsSucceeded(Pipeline pipeline, PipelineResult pipelineResult) {
     if (MetricsEnvironment.isMetricsSupported()) {
-      long expectedNumberOfAssertions = (long) PAssert.countAsserts(this);
+      long expectedNumberOfAssertions = (long) PAssert.countAsserts(pipeline);
 
       long successfulAssertions = 0;
       Iterable<MetricResult<Long>> successCounterResults =

From 731f9d9fbb466baea793a9e7376ce790a0cca98b Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 4 May 2017 09:33:41 -0700
Subject: [PATCH 036/387] GCP-IO: manage versions in top level pom

---
 pom.xml                                    | 21 +++++++++++++++++++++
 sdks/java/io/google-cloud-platform/pom.xml |  7 -------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/pom.xml b/pom.xml
index 127babdadaf5..15428913d9fa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,7 @@
     <apex.kryo.version>2.24.0</apex.kryo.version>
     <avro.version>1.8.1</avro.version>
     <bigquery.version>v2-rev295-1.22.0</bigquery.version>
+    <bigtable.version>0.9.6</bigtable.version>
     <cloudresourcemanager.version>v1-rev6-1.22.0</cloudresourcemanager.version>
     <pubsubgrpc.version>0.1.0</pubsubgrpc.version>
     <clouddebugger.version>v2-rev8-1.22.0</clouddebugger.version>
@@ -129,6 +130,7 @@
     <junit.version>4.12</junit.version>
     <mockito.version>1.9.5</mockito.version>
     <netty.version>4.1.8.Final</netty.version>
+    <netty.tcnative.version>1.1.33.Fork26</netty.tcnative.version>
     <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
     <protobuf.version>3.2.0</protobuf.version>
     <pubsub.version>v1-rev10-1.22.0</pubsub.version>
@@ -797,6 +799,25 @@
         </exclusions>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.cloud.bigtable</groupId>
+        <artifactId>bigtable-protos</artifactId>
+        <version>${bigtable.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>com.google.cloud.bigtable</groupId>
+        <artifactId>bigtable-client-core</artifactId>
+        <version>${bigtable.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-tcnative-boringssl-static</artifactId>
+        <version>${netty.tcnative.version}</version>
+        <scope>runtime</scope>
+      </dependency>
+
       <dependency>
         <groupId>com.google.cloud.datastore</groupId>
         <artifactId>datastore-v1-proto-client</artifactId>
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 9051d987f738..9c1286c4119e 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -31,10 +31,6 @@
   <description>IO library to read and write Google Cloud Platform systems from Beam.</description>
   <packaging>jar</packaging>
 
-  <properties>
-    <bigtable.version>0.9.6</bigtable.version>
-  </properties>
-
   <build>
     <plugins>
       <plugin>
@@ -158,13 +154,11 @@
     <dependency>
       <groupId>com.google.cloud.bigtable</groupId>
       <artifactId>bigtable-protos</artifactId>
-      <version>${bigtable.version}</version>
     </dependency>
 
     <dependency>
       <groupId>com.google.cloud.bigtable</groupId>
       <artifactId>bigtable-client-core</artifactId>
-      <version>${bigtable.version}</version>
     </dependency>
 
     <dependency>
@@ -220,7 +214,6 @@
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-tcnative-boringssl-static</artifactId>
-      <version>1.1.33.Fork26</version>
       <scope>runtime</scope>
     </dependency>
 

From 40dc844304a8432ee5c8e81b9cc806ef2ae3c1ea Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Wed, 3 May 2017 22:41:06 -0700
Subject: [PATCH 037/387] Remove IOChannelUtil/Factory from BigQueryIO

---
 .../beam/sdk/io/gcp/bigquery/BatchLoads.java  |  2 -
 .../sdk/io/gcp/bigquery/BigQueryHelpers.java  | 17 +++-----
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java  | 35 ++++++++-------
 .../io/gcp/bigquery/BigQuerySourceBase.java   | 14 +++---
 .../beam/sdk/io/gcp/bigquery/WriteTables.java | 43 +++++--------------
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java   | 15 +++----
 .../sdk/io/gcp/bigquery/FakeJobService.java   | 29 ++++++++-----
 7 files changed, 65 insertions(+), 90 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
index 78d39b5090fc..f4221354621b 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
@@ -205,7 +205,6 @@ public String apply(String input) {
                             bigQueryServices,
                             jobIdTokenView,
                             schemasView,
-                            stepUuid,
                             WriteDisposition.WRITE_EMPTY,
                             CreateDisposition.CREATE_IF_NEEDED,
                             dynamicDestinations))
@@ -243,7 +242,6 @@ public String apply(String input) {
                         bigQueryServices,
                         jobIdTokenView,
                         schemasView,
-                        stepUuid,
                         writeDisposition,
                         createDisposition,
                         dynamicDestinations))
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 6b4e518a5c50..318ea8910b8d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -36,12 +36,12 @@
 import java.util.regex.Matcher;
 import javax.annotation.Nullable;
 
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResolveOptions;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.util.IOChannelFactory;
-import org.apache.beam.sdk.util.IOChannelUtils;
 
 /** A set of helper functions and classes used by {@link BigQueryIO}. */
 public class BigQueryHelpers {
@@ -309,14 +309,9 @@ static TableReference createTempTableReference(String projectId, String jobUuid)
 
   static String resolveTempLocation(
       String tempLocationDir, String bigQueryOperationName, String stepUuid) {
-    try {
-      IOChannelFactory factory = IOChannelUtils.getFactory(tempLocationDir);
-      return factory.resolve(
-          factory.resolve(tempLocationDir, bigQueryOperationName),  stepUuid);
-    } catch (IOException e) {
-      throw new RuntimeException(
-          String.format("Failed to resolve temp destination directory in %s",
-              tempLocationDir), e);
-    }
+    return FileSystems.matchNewResource(tempLocationDir, true)
+        .resolve(bigQueryOperationName, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY)
+        .resolve(stepUuid, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY)
+        .toString();
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index c76ee86948d1..0e36393ea31e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -38,8 +38,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+
 import java.io.IOException;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -50,6 +50,10 @@
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.MoveOptions;
+import org.apache.beam.sdk.io.fs.ResolveOptions;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonTableRefToTableRef;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
@@ -67,8 +71,6 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.util.IOChannelFactory;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.Transport;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.KV;
@@ -520,19 +522,14 @@ void cleanup(PipelineOptions options) throws Exception {
 
               Job extractJob = getBigQueryServices().getJobService(bqOptions).getJob(jobRef);
 
-              Collection<String> extractFiles = null;
               if (extractJob != null) {
-                extractFiles = getExtractFilePaths(extractDestinationDir, extractJob);
-              } else {
-                IOChannelFactory factory = IOChannelUtils.getFactory(extractDestinationDir);
-                Collection<String> dirMatch = factory.match(extractDestinationDir);
-                if (!dirMatch.isEmpty()) {
-                  extractFiles = factory.match(factory.resolve(extractDestinationDir, "*"));
+                List<ResourceId> extractFiles =
+                    getExtractFilePaths(extractDestinationDir, extractJob);
+                if (extractFiles != null && !extractFiles.isEmpty()) {
+                  FileSystems.delete(extractFiles,
+                      MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
                 }
               }
-              if (extractFiles != null && !extractFiles.isEmpty()) {
-                IOChannelUtils.getFactory(extractFiles.iterator().next()).remove(extractFiles);
-              }
             }
           };
       return input.getPipeline()
@@ -583,7 +580,7 @@ static String getExtractDestinationUri(String extractDestinationDir) {
     return String.format("%s/%s", extractDestinationDir, "*.avro");
   }
 
-  static List<String> getExtractFilePaths(String extractDestinationDir, Job extractJob)
+  static List<ResourceId> getExtractFilePaths(String extractDestinationDir, Job extractJob)
       throws IOException {
     JobStatistics jobStats = extractJob.getStatistics();
     List<Long> counts = jobStats.getExtract().getDestinationUriFileCounts();
@@ -597,11 +594,13 @@ static List<String> getExtractFilePaths(String extractDestinationDir, Job extrac
     }
     long filesCount = counts.get(0);
 
-    ImmutableList.Builder<String> paths = ImmutableList.builder();
-    IOChannelFactory factory = IOChannelUtils.getFactory(extractDestinationDir);
+    ImmutableList.Builder<ResourceId> paths = ImmutableList.builder();
+    ResourceId extractDestinationDirResourceId =
+        FileSystems.matchNewResource(extractDestinationDir, true /* isDirectory */);
     for (long i = 0; i < filesCount; ++i) {
-      String filePath =
-          factory.resolve(extractDestinationDir, String.format("%012d%s", i, ".avro"));
+      ResourceId filePath = extractDestinationDirResourceId.resolve(
+          String.format("%012d%s", i, ".avro"),
+          ResolveOptions.StandardResolveOptions.RESOLVE_FILE);
       paths.add(filePath);
     }
     return paths.build();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index 49000d610924..945c7d40b19b 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -38,6 +38,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.io.AvroSource;
 import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -90,7 +91,7 @@ public List<BoundedSource<TableRow>> split(
           resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", stepUuid);
 
       String extractJobId = getExtractJobId(createJobIdToken(options.getJobName(), stepUuid));
-      List<String> tempFiles = executeExtract(
+      List<ResourceId> tempFiles = executeExtract(
           extractJobId, tableToExtract, jobService, bqOptions.getProject(), extractDestinationDir);
 
       TableSchema tableSchema = bqServices.getDatasetService(bqOptions)
@@ -116,7 +117,7 @@ public Coder<TableRow> getDefaultOutputCoder() {
     return TableRowJsonCoder.of();
   }
 
-  private List<String> executeExtract(
+  private List<ResourceId> executeExtract(
       String jobId, TableReference table, JobService jobService, String executingProject,
       String extractDestinationDir)
           throws InterruptedException, IOException {
@@ -143,12 +144,11 @@ private List<String> executeExtract(
 
     LOG.info("BigQuery extract job completed: {}", jobId);
 
-    List<String> tempFiles = BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob);
-    return ImmutableList.copyOf(tempFiles);
+    return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob);
   }
 
   private List<BoundedSource<TableRow>> createSources(
-      List<String> files, TableSchema tableSchema) throws IOException, InterruptedException {
+      List<ResourceId> files, TableSchema tableSchema) throws IOException, InterruptedException {
     final String jsonSchema = BigQueryIO.JSON_FACTORY.toString(tableSchema);
 
     SerializableFunction<GenericRecord, TableRow> function =
@@ -160,9 +160,9 @@ public TableRow apply(GenericRecord input) {
           }};
 
     List<BoundedSource<TableRow>> avroSources = Lists.newArrayList();
-    for (String fileName : files) {
+    for (ResourceId file : files) {
       avroSources.add(new TransformingSource<>(
-          AvroSource.from(fileName), function, getDefaultOutputCoder()));
+          AvroSource.from(file.toString()), function, getDefaultOutputCoder()));
     }
     return ImmutableList.copyOf(avroSources);
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
index c480b421c6b3..e7dba2a87a96 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
@@ -18,37 +18,30 @@
 
 package org.apache.beam.sdk.io.gcp.bigquery;
 
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation;
-
 import com.google.api.services.bigquery.model.Job;
 import com.google.api.services.bigquery.model.JobConfigurationLoad;
 import com.google.api.services.bigquery.model.JobReference;
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableSchema;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
 
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.MoveOptions;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
-import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.util.FileIOChannelFactory;
-import org.apache.beam.sdk.util.GcsIOChannelFactory;
-import org.apache.beam.sdk.util.GcsUtil;
-import org.apache.beam.sdk.util.GcsUtil.GcsUtilFactory;
-import org.apache.beam.sdk.util.IOChannelFactory;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.slf4j.Logger;
@@ -74,7 +67,6 @@ class WriteTables<DestinationT>
   private final BigQueryServices bqServices;
   private final PCollectionView<String> jobIdToken;
   private final PCollectionView<Map<DestinationT, String>> schemasView;
-  private final String stepUuid;
   private final WriteDisposition writeDisposition;
   private final CreateDisposition createDisposition;
   private final DynamicDestinations<?, DestinationT> dynamicDestinations;
@@ -84,7 +76,6 @@ public WriteTables(
       BigQueryServices bqServices,
       PCollectionView<String> jobIdToken,
       PCollectionView<Map<DestinationT, String>> schemasView,
-      String stepUuid,
       WriteDisposition writeDisposition,
       CreateDisposition createDisposition,
       DynamicDestinations<?, DestinationT> dynamicDestinations) {
@@ -92,7 +83,6 @@ public WriteTables(
     this.bqServices = bqServices;
     this.jobIdToken = jobIdToken;
     this.schemasView = schemasView;
-    this.stepUuid = stepUuid;
     this.writeDisposition = writeDisposition;
     this.createDisposition = createDisposition;
     this.dynamicDestinations = dynamicDestinations;
@@ -114,8 +104,6 @@ public void processElement(ProcessContext c) throws Exception {
           tableReference, tableDestination.getTableDescription());
     }
 
-    String tempFilePrefix = resolveTempLocation(
-        c.getPipelineOptions().getTempLocation(), "BigQueryWriteTemp", stepUuid);
     Integer partition = c.element().getKey().getShardNumber();
     List<String> partitionFiles = Lists.newArrayList(c.element().getValue());
     String jobIdPrefix =
@@ -137,7 +125,7 @@ public void processElement(ProcessContext c) throws Exception {
         tableDestination.getTableDescription());
     c.output(KV.of(tableDestination, BigQueryHelpers.toJsonString(tableReference)));
 
-    removeTemporaryFiles(c.getPipelineOptions(), tempFilePrefix, partitionFiles);
+    removeTemporaryFiles(partitionFiles);
   }
 
   private void load(
@@ -198,22 +186,11 @@ private void load(
             BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
   }
 
-  static void removeTemporaryFiles(
-      PipelineOptions options, String tempFilePrefix, Collection<String> files) throws IOException {
-    IOChannelFactory factory = IOChannelUtils.getFactory(tempFilePrefix);
-    if (factory instanceof GcsIOChannelFactory) {
-      GcsUtil gcsUtil = new GcsUtilFactory().create(options);
-      gcsUtil.remove(files);
-    } else if (factory instanceof FileIOChannelFactory) {
-      for (String filename : files) {
-        LOG.debug("Removing file {}", filename);
-        boolean exists = Files.deleteIfExists(Paths.get(filename));
-        if (!exists) {
-          LOG.debug("{} does not exist.", filename);
-        }
-      }
-    } else {
-      throw new IOException("Unrecognized file system.");
+  static void removeTemporaryFiles(Collection<String> files) throws IOException {
+    ImmutableList.Builder<ResourceId> fileResources = ImmutableList.builder();
+    for (String file: files) {
+      fileResources.add(FileSystems.matchNewResource(file, false/* isDirectory */));
     }
+    FileSystems.delete(fileResources.build(), MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 026afce4b3a6..aabae3e963c6 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -83,7 +83,9 @@
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.CountingSource;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
@@ -118,7 +120,6 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -1820,7 +1821,9 @@ public void testWriteTables() throws Exception {
         for (int k = 0; k < numFilesPerPartition; ++k) {
           String filename = Paths.get(baseDir.toString(),
               String.format("files0x%08x_%05d", tempTableId.hashCode(), k)).toString();
-          try (WritableByteChannel channel = IOChannelUtils.create(filename, MimeTypes.TEXT)) {
+          ResourceId fileResource =
+              FileSystems.matchNewResource(filename, false /* isDirectory */);
+          try (WritableByteChannel channel = FileSystems.create(fileResource, MimeTypes.TEXT)) {
             try (OutputStream output = Channels.newOutputStream(channel)) {
               TableRow tableRow = new TableRow().set("name", tableName);
               TableRowJsonCoder.of().encode(tableRow, output, Context.OUTER);
@@ -1858,7 +1861,6 @@ public void testWriteTables() throws Exception {
             fakeBqServices,
             jobIdTokenView,
             schemaMapView,
-            stepUuid,
             WriteDisposition.WRITE_EMPTY,
             CreateDisposition.CREATE_IF_NEEDED,
             new IdentityDynamicTables());
@@ -1904,14 +1906,9 @@ public void testRemoveTemporaryFiles() throws Exception {
     File tempDir = new File(bqOptions.getTempLocation());
     testNumFiles(tempDir, 10);
 
-    WriteTables.removeTemporaryFiles(bqOptions, tempFilePrefix, fileNames);
+    WriteTables.removeTemporaryFiles(fileNames);
 
     testNumFiles(tempDir, 0);
-
-    for (String fileName : fileNames) {
-      loggedWriteTables.verifyDebug("Removing file " + fileName);
-    }
-    loggedWriteTables.verifyDebug(fileNames.get(numFiles) + " does not exist.");
   }
 
   @Test
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
index 13d345ea7944..ee3af0ba9892 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
@@ -40,6 +40,7 @@
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.api.services.bigquery.model.TableSchema;
 import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
 import java.io.BufferedReader;
@@ -61,12 +62,14 @@
 import org.apache.avro.generic.GenericRecordBuilder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.MoveOptions;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
 import org.apache.beam.sdk.util.FluentBackoff;
 
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.Transport;
 import org.joda.time.Duration;
@@ -95,7 +98,7 @@ private static class JobInfo {
       HashBasedTable.create();
   private static int numExtractJobCalls = 0;
 
-  private static final com.google.common.collect.Table<String, String, List<String>>
+  private static final com.google.common.collect.Table<String, String, List<ResourceId>>
       filesForLoadJobs = HashBasedTable.create();
   private static final com.google.common.collect.Table<String, String, JobStatistics>
       dryRunQueryResults = HashBasedTable.create();
@@ -117,12 +120,17 @@ public void startLoadJob(JobReference jobRef, JobConfigurationLoad loadConfig)
       // Copy the files to a new location for import, as the temporary files will be deleted by
       // the caller.
       if (loadConfig.getSourceUris().size() > 0) {
-        List<String> loadFiles = Lists.newArrayList();
+        ImmutableList.Builder<ResourceId> sourceFiles = ImmutableList.builder();
+        ImmutableList.Builder<ResourceId> loadFiles = ImmutableList.builder();
         for (String filename : loadConfig.getSourceUris()) {
-          loadFiles.add(filename + ThreadLocalRandom.current().nextInt());
+          sourceFiles.add(FileSystems.matchNewResource(filename, false /* isDirectory */));
+          loadFiles.add(FileSystems.matchNewResource(
+              filename + ThreadLocalRandom.current().nextInt(), false /* isDirectory */));
         }
-        IOChannelUtils.getFactory(loadFiles.get(0)).copy(loadConfig.getSourceUris(), loadFiles);
-        filesForLoadJobs.put(jobRef.getProjectId(), jobRef.getJobId(), loadFiles);
+
+        FileSystems.copy(sourceFiles.build(), loadFiles.build(),
+            MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
+        filesForLoadJobs.put(jobRef.getProjectId(), jobRef.getJobId(), loadFiles.build());
       }
 
       allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job));
@@ -286,7 +294,7 @@ private JobStatus runLoadJob(JobReference jobRef, JobConfigurationLoad load)
       throws InterruptedException, IOException {
     TableReference destination = load.getDestinationTable();
     TableSchema schema = load.getSchema();
-    List<String> sourceFiles = filesForLoadJobs.get(jobRef.getProjectId(), jobRef.getJobId());
+    List<ResourceId> sourceFiles = filesForLoadJobs.get(jobRef.getProjectId(), jobRef.getJobId());
     WriteDisposition writeDisposition = WriteDisposition.valueOf(load.getWriteDisposition());
     CreateDisposition createDisposition = CreateDisposition.valueOf(load.getCreateDisposition());
     checkArgument(load.getSourceFormat().equals("NEWLINE_DELIMITED_JSON"));
@@ -298,8 +306,8 @@ private JobStatus runLoadJob(JobReference jobRef, JobConfigurationLoad load)
     datasetService.createTable(new Table().setTableReference(destination).setSchema(schema));
 
     List<TableRow> rows = Lists.newArrayList();
-    for (String filename : sourceFiles) {
-      rows.addAll(readRows(filename));
+    for (ResourceId filename : sourceFiles) {
+      rows.addAll(readRows(filename.toString()));
     }
     datasetService.insertAll(destination, rows, null);
     return new JobStatus().setState("DONE");
@@ -385,7 +393,8 @@ private long writeRows(String tableId, List<TableRow> rows, TableSchema schema,
   private void writeRowsHelper(List<TableRow> rows, Schema avroSchema,
                                String destinationPattern, int shard) throws IOException {
     String filename = destinationPattern.replace("*", String.format("%012d", shard));
-    try (WritableByteChannel channel = IOChannelUtils.create(filename, MimeTypes.BINARY);
+    try (WritableByteChannel channel = FileSystems.create(
+        FileSystems.matchNewResource(filename, false /* isDirectory */), MimeTypes.BINARY);
          DataFileWriter<GenericRecord> tableRowWriter =
              new DataFileWriter<>(new GenericDatumWriter<GenericRecord>(avroSchema))
                  .create(avroSchema, Channels.newOutputStream(channel))) {

From 55351dcebec8ba9e166c4f90555edca6b90b1b14 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 03:06:36 -0700
Subject: [PATCH 038/387] Eliminate Pipeline.getOptions

---
 .../examples/complete/game/GameStats.java     |  7 ++-
 .../complete/game/HourlyTeamScore.java        |  6 ++-
 .../examples/complete/game/LeaderBoard.java   | 26 +++++++----
 .../examples/complete/game/UserScore.java     | 20 ++++++---
 .../complete/game/utils/WriteToBigQuery.java  | 32 ++++++++------
 .../game/utils/WriteWindowedToBigQuery.java   |  8 ++--
 .../beam/runners/direct/DirectRunner.java     |  1 +
 .../runners/direct/DisplayDataValidator.java  |  6 +--
 .../dataflow/testing/TestDataflowRunner.java  |  3 +-
 .../testing/TestDataflowRunnerTest.java       | 18 +++-----
 .../beam/runners/spark/SparkRunner.java       |  4 +-
 .../runners/spark/SparkRunnerDebugger.java    |  4 +-
 .../beam/runners/spark/TestSparkRunner.java   |  5 +--
 .../spark/translation/EvaluationContext.java  | 16 +++++--
 .../translation/SparkRuntimeContext.java      |  4 +-
 .../SparkRunnerStreamingContextFactory.java   |  2 +-
 .../apache/beam/runners/spark/CacheTest.java  |  2 +-
 .../streaming/TrackStreamingSourcesTest.java  |  2 +-
 .../java/org/apache/beam/sdk/Pipeline.java    | 14 +-----
 .../apache/beam/sdk/testing/TestPipeline.java | 26 +++++------
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java  | 44 +++++++++----------
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java   |  1 +
 22 files changed, 131 insertions(+), 120 deletions(-)

diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
index a46d3c533768..abbb13bf93ee 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
@@ -24,6 +24,7 @@
 import org.apache.beam.examples.complete.game.utils.WriteWindowedToBigQuery;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
@@ -300,6 +301,8 @@ public void processElement(ProcessContext c) {
       // Write the result to BigQuery
       .apply("WriteTeamSums",
           new WriteWindowedToBigQuery<KV<String, Integer>>(
+              options.as(GcpOptions.class).getProject(),
+              options.getDataset(),
               options.getGameStatsTablePrefix() + "_team", configureWindowedWrite()));
 
 
@@ -327,7 +330,9 @@ public void processElement(ProcessContext c) {
       // Write this info to a BigQuery table.
       .apply("WriteAvgSessionLength",
              new WriteWindowedToBigQuery<Double>(
-                options.getGameStatsTablePrefix() + "_sessions", configureSessionWindowWrite()));
+                 options.as(GcpOptions.class).getProject(),
+                 options.getDataset(),
+                 options.getGameStatsTablePrefix() + "_sessions", configureSessionWindowWrite()));
     // [END DocInclude_Rewindow]
 
 
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
index 3f1ffb0de56c..2928882a1776 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
@@ -22,6 +22,7 @@
 import java.util.TimeZone;
 import org.apache.beam.examples.complete.game.utils.WriteWindowedToBigQuery;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
@@ -185,7 +186,10 @@ public static void main(String[] args) throws Exception {
       // Extract and sum teamname/score pairs from the event data.
       .apply("ExtractTeamScore", new ExtractAndSumScore("team"))
       .apply("WriteTeamScoreSums",
-        new WriteWindowedToBigQuery<KV<String, Integer>>(options.getHourlyTeamScoreTableName(),
+        new WriteWindowedToBigQuery<KV<String, Integer>>(
+            options.as(GcpOptions.class).getProject(),
+            options.getDataset(),
+            options.getHourlyTeamScoreTableName(),
             configureWindowedTableWrite()));
 
 
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
index 9af34c536e80..bfad9f65fc3f 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
@@ -27,6 +27,7 @@
 import org.apache.beam.examples.complete.game.utils.WriteWindowedToBigQuery;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
@@ -194,14 +195,20 @@ public static void main(String[] args) throws Exception {
             .withTimestampAttribute(TIMESTAMP_ATTRIBUTE).fromTopic(options.getTopic()))
         .apply("ParseGameEvent", ParDo.of(new ParseEventFn()));
 
-    gameEvents.apply("CalculateTeamScores",
-        new CalculateTeamScores(
-            Duration.standardMinutes(options.getTeamWindowDuration()),
-            Duration.standardMinutes(options.getAllowedLateness())))
+    gameEvents
+        .apply(
+            "CalculateTeamScores",
+            new CalculateTeamScores(
+                Duration.standardMinutes(options.getTeamWindowDuration()),
+                Duration.standardMinutes(options.getAllowedLateness())))
         // Write the results to BigQuery.
-        .apply("WriteTeamScoreSums",
-               new WriteWindowedToBigQuery<KV<String, Integer>>(
-                  options.getLeaderBoardTableName() + "_team", configureWindowedTableWrite()));
+        .apply(
+            "WriteTeamScoreSums",
+            new WriteWindowedToBigQuery<KV<String, Integer>>(
+                options.as(GcpOptions.class).getProject(),
+                options.getDataset(),
+                options.getLeaderBoardTableName() + "_team",
+                configureWindowedTableWrite()));
     gameEvents
         .apply(
             "CalculateUserScores",
@@ -210,7 +217,10 @@ public static void main(String[] args) throws Exception {
         .apply(
             "WriteUserScoreSums",
             new WriteToBigQuery<KV<String, Integer>>(
-                options.getLeaderBoardTableName() + "_user", configureGlobalWindowBigQueryWrite()));
+                options.as(GcpOptions.class).getProject(),
+                options.getDataset(),
+                options.getLeaderBoardTableName() + "_user",
+                configureGlobalWindowBigQueryWrite()));
 
     // Run the pipeline and wait for the pipeline to finish; capture cancellation requests from the
     // command line.
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index c136c2e3de38..8110146d7020 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -24,6 +24,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.DefaultCoder;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
@@ -226,13 +227,18 @@ public static void main(String[] args) throws Exception {
     Pipeline pipeline = Pipeline.create(options);
 
     // Read events from a text file and parse them.
-    pipeline.apply(TextIO.read().from(options.getInput()))
-      .apply("ParseGameEvent", ParDo.of(new ParseEventFn()))
-      // Extract and sum username/score pairs from the event data.
-      .apply("ExtractUserScore", new ExtractAndSumScore("user"))
-      .apply("WriteUserScoreSums",
-          new WriteToBigQuery<KV<String, Integer>>(options.getUserScoreTableName(),
-                                                   configureBigQueryWrite()));
+    pipeline
+        .apply(TextIO.read().from(options.getInput()))
+        .apply("ParseGameEvent", ParDo.of(new ParseEventFn()))
+        // Extract and sum username/score pairs from the event data.
+        .apply("ExtractUserScore", new ExtractAndSumScore("user"))
+        .apply(
+            "WriteUserScoreSums",
+            new WriteToBigQuery<KV<String, Integer>>(
+                options.as(GcpOptions.class).getProject(),
+                options.getDataset(),
+                options.getUserScoreTableName(),
+                configureBigQueryWrite()));
 
     // Run the batch pipeline.
     pipeline.run().waitUntilFinish();
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToBigQuery.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToBigQuery.java
index f767d21b6fb5..2ec4e5c9a133 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToBigQuery.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToBigQuery.java
@@ -25,13 +25,9 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import org.apache.beam.examples.complete.game.UserScore;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
-import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -47,14 +43,21 @@
 public class WriteToBigQuery<InputT>
     extends PTransform<PCollection<InputT>, PDone> {
 
+  protected String projectId;
+  protected String datasetId;
   protected String tableName;
   protected Map<String, FieldInfo<InputT>> fieldInfo;
 
   public WriteToBigQuery() {
   }
 
-  public WriteToBigQuery(String tableName,
+  public WriteToBigQuery(
+      String projectId,
+      String datasetId,
+      String tableName,
       Map<String, FieldInfo<InputT>> fieldInfo) {
+    this.projectId = projectId;
+    this.datasetId = datasetId;
     this.tableName = tableName;
     this.fieldInfo = fieldInfo;
   }
@@ -120,20 +123,21 @@ protected TableSchema getSchema() {
   @Override
   public PDone expand(PCollection<InputT> teamAndScore) {
     teamAndScore
-      .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
-      .apply(BigQueryIO.writeTableRows().to(getTable(teamAndScore.getPipeline(), tableName))
-          .withSchema(getSchema())
-          .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
-          .withWriteDisposition(WriteDisposition.WRITE_APPEND));
+        .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
+        .apply(
+            BigQueryIO.writeTableRows()
+                .to(getTable(projectId, datasetId, tableName))
+                .withSchema(getSchema())
+                .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
+                .withWriteDisposition(WriteDisposition.WRITE_APPEND));
     return PDone.in(teamAndScore.getPipeline());
   }
 
   /** Utility to construct an output table reference. */
-  static TableReference getTable(Pipeline pipeline, String tableName) {
-    PipelineOptions options = pipeline.getOptions();
+  static TableReference getTable(String projectId, String datasetId, String tableName) {
     TableReference table = new TableReference();
-    table.setDatasetId(options.as(UserScore.Options.class).getDataset());
-    table.setProjectId(options.as(GcpOptions.class).getProject());
+    table.setDatasetId(datasetId);
+    table.setProjectId(projectId);
     table.setTableId(tableName);
     return table;
   }
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java
index e602258c74fa..deb9db2c0fd5 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java
@@ -37,9 +37,9 @@
 public class WriteWindowedToBigQuery<T>
     extends WriteToBigQuery<T> {
 
-  public WriteWindowedToBigQuery(String tableName,
-      Map<String, FieldInfo<T>> fieldInfo) {
-    super(tableName, fieldInfo);
+  public WriteWindowedToBigQuery(
+      String projectId, String datasetId, String tableName, Map<String, FieldInfo<T>> fieldInfo) {
+    super(projectId, datasetId, tableName, fieldInfo);
   }
 
   /** Convert each key/score pair into a BigQuery TableRow. */
@@ -62,7 +62,7 @@ public PDone expand(PCollection<T> teamAndScore) {
     teamAndScore
       .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
       .apply(BigQueryIO.writeTableRows()
-                .to(getTable(teamAndScore.getPipeline(), tableName))
+                .to(getTable(projectId, datasetId, tableName))
                 .withSchema(getSchema())
                 .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
                 .withWriteDisposition(WriteDisposition.WRITE_APPEND));
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index c6168b3e6603..984598a6faaf 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -163,6 +163,7 @@ public DirectPipelineResult run(Pipeline pipeline) {
     pipeline.traverseTopologically(keyedPValueVisitor);
 
     DisplayDataValidator.validatePipeline(pipeline);
+    DisplayDataValidator.validateOptions(getPipelineOptions());
 
     DirectGraph graph = graphVisitor.getGraph();
     EvaluationContext context =
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java
index c77cb48f4f9b..209c8010bcab 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DisplayDataValidator.java
@@ -18,6 +18,7 @@
 package org.apache.beam.runners.direct;
 
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
@@ -31,12 +32,11 @@ class DisplayDataValidator {
   private DisplayDataValidator() {}
 
   static void validatePipeline(Pipeline pipeline) {
-    validateOptions(pipeline);
     validateTransforms(pipeline);
   }
 
-  private static void validateOptions(Pipeline pipeline) {
-    evaluateDisplayData(pipeline.getOptions());
+  static void validateOptions(PipelineOptions options) {
+    evaluateDisplayData(options);
   }
 
   private static void validateTransforms(Pipeline pipeline) {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
index ba9d971646b3..c238d80bc933 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
@@ -98,7 +98,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
   DataflowPipelineJob run(Pipeline pipeline, DataflowRunner runner) {
     updatePAssertCount(pipeline);
 
-    TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
+    TestPipelineOptions testPipelineOptions = options.as(TestPipelineOptions.class);
     final DataflowPipelineJob job;
     job = runner.run(pipeline);
 
@@ -188,7 +188,6 @@ DataflowPipelineJob run(Pipeline pipeline, DataflowRunner runner) {
 
   @VisibleForTesting
   void updatePAssertCount(Pipeline pipeline) {
-    DataflowPipelineOptions options = pipeline.getOptions().as(DataflowPipelineOptions.class);
     if (DataflowRunner.hasExperiment(options, "beam_fn_api")) {
       // TODO[BEAM-1866]: FnAPI does not support metrics, so expect 0 assertions.
       expectedNumberOfAssertions = 0;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
index 54eb88d04f87..eb068e67c3b3 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
@@ -464,8 +464,7 @@ public void testBatchOnCreateMatcher() throws Exception {
     when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob);
 
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
-    p.getOptions().as(TestPipelineOptions.class)
-        .setOnCreateMatcher(new TestSuccessMatcher(mockJob, 0));
+    options.as(TestPipelineOptions.class).setOnCreateMatcher(new TestSuccessMatcher(mockJob, 0));
 
     when(mockClient.getJobMetrics(anyString()))
         .thenReturn(generateMockMetricResponse(true /* success */, true /* tentative */));
@@ -488,8 +487,7 @@ public void testStreamingOnCreateMatcher() throws Exception {
     when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob);
 
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
-    p.getOptions().as(TestPipelineOptions.class)
-        .setOnCreateMatcher(new TestSuccessMatcher(mockJob, 0));
+    options.as(TestPipelineOptions.class).setOnCreateMatcher(new TestSuccessMatcher(mockJob, 0));
 
     when(mockJob.waitUntilFinish(any(Duration.class), any(JobMessagesHandler.class)))
         .thenReturn(State.DONE);
@@ -515,8 +513,7 @@ public void testBatchOnSuccessMatcherWhenPipelineSucceeds() throws Exception {
     when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob);
 
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
-    p.getOptions().as(TestPipelineOptions.class)
-        .setOnSuccessMatcher(new TestSuccessMatcher(mockJob, 1));
+    options.as(TestPipelineOptions.class).setOnSuccessMatcher(new TestSuccessMatcher(mockJob, 1));
 
     when(mockClient.getJobMetrics(anyString()))
         .thenReturn(generateMockMetricResponse(true /* success */, true /* tentative */));
@@ -544,8 +541,7 @@ public void testStreamingOnSuccessMatcherWhenPipelineSucceeds() throws Exception
     when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob);
 
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
-    p.getOptions().as(TestPipelineOptions.class)
-        .setOnSuccessMatcher(new TestSuccessMatcher(mockJob, 1));
+    options.as(TestPipelineOptions.class).setOnSuccessMatcher(new TestSuccessMatcher(mockJob, 1));
 
     when(mockJob.waitUntilFinish(any(Duration.class), any(JobMessagesHandler.class)))
         .thenReturn(State.DONE);
@@ -570,8 +566,7 @@ public void testBatchOnSuccessMatcherWhenPipelineFails() throws Exception {
     when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob);
 
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
-    p.getOptions().as(TestPipelineOptions.class)
-        .setOnSuccessMatcher(new TestFailureMatcher());
+    options.as(TestPipelineOptions.class).setOnSuccessMatcher(new TestFailureMatcher());
 
     when(mockClient.getJobMetrics(anyString()))
         .thenReturn(generateMockMetricResponse(false /* success */, true /* tentative */));
@@ -606,8 +601,7 @@ public void testStreamingOnSuccessMatcherWhenPipelineFails() throws Exception {
     when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob);
 
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
-    p.getOptions().as(TestPipelineOptions.class)
-        .setOnSuccessMatcher(new TestFailureMatcher());
+    options.as(TestPipelineOptions.class).setOnSuccessMatcher(new TestFailureMatcher());
 
     when(mockJob.waitUntilFinish(any(Duration.class), any(JobMessagesHandler.class)))
         .thenReturn(State.FAILED);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 97487f3f0277..1a0c04234da2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -193,7 +193,7 @@ public void run() {
     } else {
       // create the evaluation context
       final JavaSparkContext jsc = SparkContextFactory.getSparkContext(mOptions);
-      final EvaluationContext evaluationContext = new EvaluationContext(jsc, pipeline);
+      final EvaluationContext evaluationContext = new EvaluationContext(jsc, pipeline, mOptions);
       translator = new TransformTranslator.Translator();
 
       // update the cache candidates
@@ -383,7 +383,7 @@ protected boolean shouldDefer(TransformHierarchy.Node node) {
         LOG.info(
             "Deferring combine transformation {} for job {}",
             transform,
-            ctxt.getPipeline().getOptions().getJobName());
+            ctxt.getOptions().getJobName());
         return true;
       }
       // default.
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
index 7f7aefc2b6b6..8d47e1a31e82 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
@@ -89,10 +89,10 @@ public SparkPipelineResult run(Pipeline pipeline) {
         && ((TestSparkPipelineOptions) options).isForceStreaming()) {
       SparkPipelineTranslator streamingTranslator =
           new StreamingTransformTranslator.Translator(translator);
-      EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, jssc);
+      EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, options, jssc);
       visitor = new SparkNativePipelineVisitor(streamingTranslator, ctxt);
     } else {
-      EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, jssc);
+      EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, options, jssc);
       visitor = new SparkNativePipelineVisitor(translator, ctxt);
     }
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 1e678131d92b..6808d7b6b132 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -83,6 +83,7 @@
 public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestSparkRunner.class);
+  private final TestSparkPipelineOptions testSparkPipelineOptions;
 
   private SparkRunner delegate;
   private boolean isForceStreaming;
@@ -90,6 +91,7 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
   private TestSparkRunner(TestSparkPipelineOptions options) {
     this.delegate = SparkRunner.fromOptions(options);
     this.isForceStreaming = options.isForceStreaming();
+    this.testSparkPipelineOptions = options;
   }
 
   public static TestSparkRunner fromOptions(PipelineOptions options) {
@@ -101,9 +103,6 @@ public static TestSparkRunner fromOptions(PipelineOptions options) {
 
   @Override
   public SparkPipelineResult run(Pipeline pipeline) {
-    TestSparkPipelineOptions testSparkPipelineOptions =
-        pipeline.getOptions().as(TestSparkPipelineOptions.class);
-    //
     // if the pipeline forces execution as a streaming pipeline,
     // and the source is an adapted unbounded source (as bounded),
     // read it as unbounded source via UnboundedReadFromBoundedSource.
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 838c504a1aad..5d77e9162e7d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -30,6 +30,7 @@
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -57,15 +58,18 @@ public class EvaluationContext {
   private AppliedPTransform<?, ?, ?> currentTransform;
   private final SparkPCollectionView pviews = new SparkPCollectionView();
   private final Map<PCollection, Long> cacheCandidates = new HashMap<>();
+  private final PipelineOptions options;
 
-  public EvaluationContext(JavaSparkContext jsc, Pipeline pipeline) {
+  public EvaluationContext(JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options) {
     this.jsc = jsc;
     this.pipeline = pipeline;
-    this.runtime = new SparkRuntimeContext(pipeline);
+    this.options = options;
+    this.runtime = new SparkRuntimeContext(pipeline, options);
   }
 
-  public EvaluationContext(JavaSparkContext jsc, Pipeline pipeline, JavaStreamingContext jssc) {
-    this(jsc, pipeline);
+  public EvaluationContext(
+      JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options, JavaStreamingContext jssc) {
+    this(jsc, pipeline, options);
     this.jssc = jssc;
   }
 
@@ -81,6 +85,10 @@ public Pipeline getPipeline() {
     return pipeline;
   }
 
+  public PipelineOptions getOptions() {
+    return options;
+  }
+
   public SparkRuntimeContext getRuntimeContext() {
     return runtime;
   }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index e006143cc0f3..3db1ab51ddbb 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -35,8 +35,8 @@ public class SparkRuntimeContext implements Serializable {
   private final String serializedPipelineOptions;
   private transient CoderRegistry coderRegistry;
 
-  SparkRuntimeContext(Pipeline pipeline) {
-    this.serializedPipelineOptions = serializePipelineOptions(pipeline.getOptions());
+  SparkRuntimeContext(Pipeline pipeline, PipelineOptions options) {
+    this.serializedPipelineOptions = serializePipelineOptions(options);
   }
 
   private String serializePipelineOptions(PipelineOptions pipelineOptions) {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java
index 2dd18f395b0c..6a153ffd6d40 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java
@@ -82,7 +82,7 @@ public JavaStreamingContext call() throws Exception {
     // We must first init accumulators since translators expect them to be instantiated.
     SparkRunner.initAccumulators(options, jsc);
 
-    EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, jssc);
+    EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, options, jssc);
     // update cache candidates
     SparkRunner.updateCacheCandidates(pipeline, translator, ctxt);
     pipeline.traverseTopologically(new SparkRunner.Evaluator(translator, ctxt));
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java
index 24b2e7b3a358..d3d0823c0470 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/CacheTest.java
@@ -51,7 +51,7 @@ public void cacheCandidatesUpdaterTest() throws Exception {
     pCollection.apply(Count.<String>globally());
 
     JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
-    EvaluationContext ctxt = new EvaluationContext(jsc, pipeline);
+    EvaluationContext ctxt = new EvaluationContext(jsc, pipeline, options);
     SparkRunner.CacheVisitor cacheVisitor =
         new SparkRunner.CacheVisitor(new TransformTranslator.Translator(), ctxt);
     pipeline.traverseTopologically(cacheVisitor);
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index 41ccd0837dcb..3dcab26e946b 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -135,7 +135,7 @@ private StreamingSourceTracker(
         Pipeline pipeline,
         Class<? extends PTransform> transformClassToAssert,
         Integer... expected) {
-      this.ctxt = new EvaluationContext(jssc.sparkContext(), pipeline, jssc);
+      this.ctxt = new EvaluationContext(jssc.sparkContext(), pipeline, options, jssc);
       this.evaluator = new SparkRunner.Evaluator(
           new StreamingTransformTranslator.Translator(new TransformTranslator.Translator()), ctxt);
       this.transformClassToAssert = transformClassToAssert;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index 6fa7a5a3b4e5..f7c3f24c46bd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -460,7 +460,7 @@ OutputT applyTransform(String name, InputT input,
   private Set<String> usedFullNames = new HashSet<>();
   private CoderRegistry coderRegistry;
   private final List<String> unstableNames = new ArrayList<>();
-  protected final PipelineOptions defaultOptions;
+  private final PipelineOptions defaultOptions;
 
   protected Pipeline(PipelineOptions options) {
     this.defaultOptions = options;
@@ -471,18 +471,6 @@ public String toString() {
     return "Pipeline#" + hashCode();
   }
 
-  /**
-   * Returns the default {@link PipelineOptions} provided to {@link #create(PipelineOptions)}.
-   *
-   * @deprecated see BEAM-818 Remove Pipeline.getPipelineOptions. Configuration should be explicitly
-   *     provided to a transform if it is required.
-   */
-  @Deprecated
-  public PipelineOptions getOptions() {
-    return defaultOptions;
-  }
-
-
   /**
    * Applies a {@link PTransform} to the given {@link PInput}.
    *
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index 2d34b2249467..96cae51765e6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -26,7 +26,6 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
@@ -104,6 +103,8 @@
  */
 public class TestPipeline extends Pipeline implements TestRule {
 
+  private final PipelineOptions options;
+
   private static class PipelineRunEnforcement {
 
     @SuppressWarnings("WeakerAccess")
@@ -183,10 +184,7 @@ private boolean isEmptyPipeline(final Pipeline pipeline) {
     private void verifyPipelineExecution() {
       if (!isEmptyPipeline(pipeline)) {
         if (!runAttempted && !enableAutoRunIfMissing) {
-          throw new PipelineRunMissingException(
-              "The pipeline has not been run (runner: "
-                  + pipeline.getOptions().getRunner().getSimpleName()
-                  + ")");
+          throw new PipelineRunMissingException("The pipeline has not been run.");
 
         } else {
           final List<TransformHierarchy.Node> pipelineNodes = recordPipelineNodes(pipeline);
@@ -272,6 +270,11 @@ public static TestPipeline fromOptions(PipelineOptions options) {
 
   private TestPipeline(final PipelineOptions options) {
     super(options);
+    this.options = options;
+  }
+
+  public PipelineOptions getOptions() {
+    return this.options;
   }
 
   @Override
@@ -288,7 +291,7 @@ private void setDeducedEnforcementLevel() {
                   .anyMatch(Annotations.Predicates.isCategoryOf(NeedsRunner.class, true));
 
           final boolean crashingRunner =
-              CrashingRunner.class.isAssignableFrom(getOptions().getRunner());
+              CrashingRunner.class.isAssignableFrom(options.getRunner());
 
           checkState(
               !(annotatedWithNeedsRunner && crashingRunner),
@@ -381,18 +384,9 @@ public TestPipeline enableAutoRunIfMissing(final boolean enable) {
     return this;
   }
 
-  @VisibleForTesting
-  @Override
-  /**
-   * Get this pipeline's options.
-   */
-  public PipelineOptions getOptions() {
-    return defaultOptions;
-  }
-
   @Override
   public String toString() {
-    return "TestPipeline#" + getOptions().as(ApplicationNameOptions.class).getAppName();
+    return "TestPipeline#" + options.as(ApplicationNameOptions.class).getAppName();
   }
 
   /** Creates {@link PipelineOptions} for testing. */
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 0e36393ea31e..fbbf862d1e84 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -888,6 +888,26 @@ Write<T> withTestServices(BigQueryServices testServices) {
     public void validate(PipelineOptions pipelineOptions) {
       BigQueryOptions options = pipelineOptions.as(BigQueryOptions.class);
 
+      // The user specified a table.
+      if (getJsonTableRef() != null && getJsonTableRef().isAccessible() && getValidate()) {
+        TableReference table = getTableWithDefaultProject(options).get();
+        DatasetService datasetService = getBigQueryServices().getDatasetService(options);
+        // Check for destination table presence and emptiness for early failure notification.
+        // Note that a presence check can fail when the table or dataset is created by an earlier
+        // stage of the pipeline. For these cases the #withoutValidation method can be used to
+        // disable the check.
+        BigQueryHelpers.verifyDatasetPresence(datasetService, table);
+        if (getCreateDisposition() == BigQueryIO.Write.CreateDisposition.CREATE_NEVER) {
+          BigQueryHelpers.verifyTablePresence(datasetService, table);
+        }
+        if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
+          BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table);
+        }
+      }
+    }
+
+    @Override
+    public WriteResult expand(PCollection<T> input) {
       // We must have a destination to write to!
       checkState(
           getTableFunction() != null || getJsonTableRef() != null
@@ -916,29 +936,7 @@ public void validate(PipelineOptions pipelineOptions) {
       checkArgument(2
               > Iterables.size(Iterables.filter(allSchemaArgs, Predicates.notNull())),
           "No more than one of jsonSchema, schemaFromView, or dynamicDestinations may "
-          + "be set");
-
-      // The user specified a table.
-      if (getJsonTableRef() != null && getJsonTableRef().isAccessible() && getValidate()) {
-        TableReference table = getTableWithDefaultProject(options).get();
-        DatasetService datasetService = getBigQueryServices().getDatasetService(options);
-        // Check for destination table presence and emptiness for early failure notification.
-        // Note that a presence check can fail when the table or dataset is created by an earlier
-        // stage of the pipeline. For these cases the #withoutValidation method can be used to
-        // disable the check.
-        BigQueryHelpers.verifyDatasetPresence(datasetService, table);
-        if (getCreateDisposition() == BigQueryIO.Write.CreateDisposition.CREATE_NEVER) {
-          BigQueryHelpers.verifyTablePresence(datasetService, table);
-        }
-        if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
-          BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table);
-        }
-      }
-    }
-
-    @Override
-    public WriteResult expand(PCollection<T> input) {
-      validate(input.getPipeline().getOptions());
+              + "be set");
 
       DynamicDestinations<T, ?> dynamicDestinations = getDynamicDestinations();
       if (dynamicDestinations == null) {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index aabae3e963c6..b893ad5a5189 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -1175,6 +1175,7 @@ public TableRow apply(Long input) {
                 .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
                 .withSchema(new TableSchema())
                 .withTestServices(fakeBqServices));
+    p.run();
   }
 
   @Test

From 7f19e7a0653e6e7f0d53ab537bc4b00f0ab486c0 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 4 May 2017 12:01:53 -0700
Subject: [PATCH 039/387] Cleanup old cython files before testing again

---
 sdks/python/tox.ini | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 9c40eb5af8ce..2592b17501ae 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -48,6 +48,11 @@ deps =
 whitelist_externals=find
 commands =
   python --version
+  # Clean up all previous cython generated files.
+  - find apache_beam -type f -name '*.c' -delete
+  - find apache_beam -type f -name '*.so' -delete
+  - find target/build -type f -name '*.c' -delete
+  - find target/build -type f -name '*.so' -delete
   pip install -e .[test]
   python apache_beam/examples/complete/autocomplete_test.py
   python setup.py test

From 4f934923d28798dfe7cd18c86ff4bcf8eebc27e5 Mon Sep 17 00:00:00 2001
From: JingsongLi <lzljs3620320@aliyun.com>
Date: Mon, 20 Mar 2017 12:12:31 +0800
Subject: [PATCH 040/387] [BEAM-1727] Add align and offset to Timer

---
 .../construction/PTransformMatchersTest.java  |   2 +-
 .../beam/runners/core/SimpleDoFnRunner.java   |  44 ++++++-
 .../runners/core/SimpleDoFnRunnerTest.java    |   2 +-
 .../org/apache/beam/sdk/transforms/DoFn.java  |   2 +-
 .../java/org/apache/beam/sdk/util/Timer.java  |  27 +++--
 .../apache/beam/sdk/transforms/ParDoTest.java | 113 +++++++++++++++++-
 6 files changed, 171 insertions(+), 19 deletions(-)

diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index e7d4c64805c0..bb1b1cd63184 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -193,7 +193,7 @@ public void processElement(ProcessContext c, @StateId(stateId) ValueState<Intege
 
         @ProcessElement
         public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
-          timer.setForNowPlus(Duration.standardSeconds(1));
+          timer.offset(Duration.standardSeconds(1)).setRelative();
           context.output(3);
         }
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 8a3e25fec0a5..7f29a6fff52e 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -788,6 +788,8 @@ private static class TimerInternalsTimer implements Timer {
     private final StateNamespace namespace;
     private final String timerId;
     private final TimerSpec spec;
+    private Duration period = Duration.ZERO;
+    private Duration offset = Duration.ZERO;
 
     public TimerInternalsTimer(
         BoundedWindow window,
@@ -812,12 +814,45 @@ public void set(Instant target) {
     }
 
     @Override
-    public void setForNowPlus(Duration durationFromNow) {
-      Instant target = getCurrentTime().plus(durationFromNow);
-      verifyTargetTime(target);
+    public void setRelative() {
+      Instant target;
+      Instant now = getCurrentTime();
+      if (period.equals(Duration.ZERO)) {
+        target = now.plus(offset);
+      } else {
+        long millisSinceStart = now.plus(offset).getMillis() % period.getMillis();
+        target = millisSinceStart == 0 ? now : now.plus(period).minus(millisSinceStart);
+      }
+      target = minTargetAndGcTime(target);
       setUnderlyingTimer(target);
     }
 
+    @Override
+    public Timer offset(Duration offset) {
+      this.offset = offset;
+      return this;
+    }
+
+    @Override
+    public Timer align(Duration period) {
+      this.period = period;
+      return this;
+    }
+
+    /**
+     * For event time timers the target time should be prior to window GC time. So it return
+     * min(time to set, GC Time of window).
+     */
+    private Instant minTargetAndGcTime(Instant target) {
+      if (TimeDomain.EVENT_TIME.equals(spec.getTimeDomain())) {
+        Instant windowExpiry = window.maxTimestamp().plus(allowedLateness);
+        if (target.isAfter(windowExpiry)) {
+          return windowExpiry;
+        }
+      }
+      return target;
+    }
+
     /**
      * Ensures that the target time is reasonable. For event time timers this means that the
      * time should be prior to window GC time.
@@ -836,7 +871,7 @@ private void verifyAbsoluteTimeDomain() {
       if (!TimeDomain.EVENT_TIME.equals(spec.getTimeDomain())) {
         throw new IllegalStateException(
             "Cannot only set relative timers in processing time domain."
-                + " Use #setForNowPlus(Duration)");
+                + " Use #setRelative()");
       }
     }
 
@@ -867,5 +902,6 @@ private Instant getCurrentTime() {
               String.format("Timer created for unknown time domain %s", spec.getTimeDomain()));
       }
     }
+
   }
 }
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index 3e404ad2add8..9b63bab94c1e 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -388,7 +388,7 @@ private static class DoFnWithTimers<W extends BoundedWindow> extends DoFn<String
 
     @ProcessElement
     public void process(ProcessContext context, @TimerId(TIMER_ID) Timer timer) {
-      timer.setForNowPlus(TIMER_OFFSET);
+      timer.offset(TIMER_OFFSET).setRelative();
     }
 
     @OnTimer(TIMER_ID)
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 036847672e25..eab08f6b8793 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -385,7 +385,7 @@ public interface OutputReceiver<T> {
    *    public void processElement(
    *        ProcessContext c,
    *       {@literal @TimerId("my-timer-id") Timer myTimer}) {
-   *      myTimer.setForNowPlus(Duration.standardSeconds(...));
+   *      myTimer.offset(Duration.standardSeconds(...)).setRelative();
    *    }
    *
    *   {@literal @OnTimer("my-timer-id")}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timer.java
index 45a2a6623297..9727969fcc28 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timer.java
@@ -49,19 +49,30 @@ public interface Timer {
    *
    * <p>For {@link TimeDomain#PROCESSING_TIME}, the behavior is be unpredictable, since processing
    * time timers are ignored after a window has expired. Instead, it is recommended to use
-   * {@link #setForNowPlus(Duration)}.
+   * {@link #setRelative()}.
    */
-  void set(Instant instant);
-
-  /**
-   * Sets or resets the time relative to the current time in the timer's {@link TimeDomain} at which
-   * this it should fire. If the timer was already set, resets it to the new requested time.
-   */
-  void setForNowPlus(Duration durationFromNow);
+  void set(Instant absoluteTime);
 
   /**
    * Unsets this timer. It is permitted to {@code cancel()} whether or not the timer was actually
    * set.
    */
   void cancel();
+
+  /**
+   * Sets the timer relative to the current time, according to any offset and alignment specified.
+   * Using {@link #offset(Duration)} and {@link #align(Duration)}.
+   */
+  void setRelative();
+
+  /**
+   * Set the align offset.
+   */
+  Timer offset(Duration offset);
+
+  /**
+   * Aligns a timestamp to the next boundary of {@code period}.
+   */
+  Timer align(Duration period);
+
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index d4475c925266..1c919d4fb659 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -2582,7 +2582,7 @@ public void testEventTimeTimerBounded() throws Exception {
 
           @ProcessElement
           public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
-            timer.setForNowPlus(Duration.standardSeconds(1));
+            timer.offset(Duration.standardSeconds(1)).setRelative();
             context.output(3);
           }
 
@@ -2597,6 +2597,36 @@ public void onTimer(OnTimerContext context) {
     pipeline.run();
   }
 
+  @Test
+  @Category({ValidatesRunner.class, UsesTimersInParDo.class})
+  public void testEventTimeTimerAlignBounded() throws Exception {
+    final String timerId = "foo";
+
+    DoFn<KV<String, Integer>, KV<Integer, Instant>> fn =
+        new DoFn<KV<String, Integer>, KV<Integer, Instant>>() {
+
+          @TimerId(timerId)
+          private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+          @ProcessElement
+          public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
+            timer.align(Duration.standardSeconds(1)).offset(Duration.millis(1)).setRelative();
+            context.output(KV.of(3, context.timestamp()));
+          }
+
+          @OnTimer(timerId)
+          public void onTimer(OnTimerContext context) {
+            context.output(KV.of(42, context.timestamp()));
+          }
+        };
+
+    PCollection<KV<Integer, Instant>> output =
+        pipeline.apply(Create.of(KV.of("hello", 37))).apply(ParDo.of(fn));
+    PAssert.that(output).containsInAnyOrder(KV.of(3, BoundedWindow.TIMESTAMP_MIN_VALUE),
+        KV.of(42, BoundedWindow.TIMESTAMP_MIN_VALUE.plus(1774)));
+    pipeline.run();
+  }
+
   @Test
   @Category({ValidatesRunner.class, UsesTimersInParDo.class})
   public void testTimerReceivedInOriginalWindow() throws Exception {
@@ -2610,7 +2640,7 @@ public void testTimerReceivedInOriginalWindow() throws Exception {
 
           @ProcessElement
           public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
-            timer.setForNowPlus(Duration.standardSeconds(1));
+            timer.offset(Duration.standardSeconds(1)).setRelative();
           }
 
           @OnTimer(timerId)
@@ -2814,7 +2844,7 @@ public void testSimpleProcessingTimerTimer() throws Exception {
 
           @ProcessElement
           public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
-            timer.setForNowPlus(Duration.standardSeconds(1));
+            timer.offset(Duration.standardSeconds(1)).setRelative();
             context.output(3);
           }
 
@@ -2848,7 +2878,7 @@ public void testEventTimeTimerUnbounded() throws Exception {
 
           @ProcessElement
           public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
-            timer.setForNowPlus(Duration.standardSeconds(1));
+            timer.offset(Duration.standardSeconds(1)).setRelative();
             context.output(3);
           }
 
@@ -2870,6 +2900,81 @@ public void onTimer(OnTimerContext context) {
     pipeline.run();
   }
 
+  @Test
+  @Category({NeedsRunner.class, UsesTimersInParDo.class, UsesTestStream.class})
+  public void testEventTimeTimerAlignUnbounded() throws Exception {
+    final String timerId = "foo";
+
+    DoFn<KV<String, Integer>, KV<Integer, Instant>> fn =
+        new DoFn<KV<String, Integer>, KV<Integer, Instant>>() {
+
+          @TimerId(timerId)
+          private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+          @ProcessElement
+          public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
+            timer.align(Duration.standardSeconds(1)).offset(Duration.millis(1)).setRelative();
+            context.output(KV.of(3, context.timestamp()));
+          }
+
+          @OnTimer(timerId)
+          public void onTimer(OnTimerContext context) {
+            context.output(KV.of(42, context.timestamp()));
+          }
+        };
+
+    TestStream<KV<String, Integer>> stream = TestStream.create(KvCoder
+        .of(StringUtf8Coder.of(), VarIntCoder.of()))
+        .advanceWatermarkTo(new Instant(5))
+        .addElements(KV.of("hello", 37))
+        .advanceWatermarkTo(new Instant(0).plus(Duration.standardSeconds(1).plus(1)))
+        .advanceWatermarkToInfinity();
+
+    PCollection<KV<Integer, Instant>> output = pipeline.apply(stream).apply(ParDo.of(fn));
+    PAssert.that(output).containsInAnyOrder(KV.of(3, new Instant(5)),
+        KV.of(42, new Instant(Duration.standardSeconds(1).minus(1).getMillis())));
+    pipeline.run();
+  }
+
+  @Test
+  @Category({NeedsRunner.class, UsesTimersInParDo.class, UsesTestStream.class})
+  public void testEventTimeTimerAlignAfterGcTimeUnbounded() throws Exception {
+    final String timerId = "foo";
+
+    DoFn<KV<String, Integer>, KV<Integer, Instant>> fn =
+        new DoFn<KV<String, Integer>, KV<Integer, Instant>>() {
+
+          @TimerId(timerId)
+          private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+          @ProcessElement
+          public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
+            // This aligned time will exceed the END_OF_GLOBAL_WINDOW
+            timer.align(Duration.standardDays(1)).setRelative();
+            context.output(KV.of(3, context.timestamp()));
+          }
+
+          @OnTimer(timerId)
+          public void onTimer(OnTimerContext context) {
+            context.output(KV.of(42, context.timestamp()));
+          }
+        };
+
+    TestStream<KV<String, Integer>> stream = TestStream.create(KvCoder
+        .of(StringUtf8Coder.of(), VarIntCoder.of()))
+        // See GlobalWindow,
+        // END_OF_GLOBAL_WINDOW is TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1))
+        .advanceWatermarkTo(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1)))
+        .addElements(KV.of("hello", 37))
+        .advanceWatermarkToInfinity();
+
+    PCollection<KV<Integer, Instant>> output = pipeline.apply(stream).apply(ParDo.of(fn));
+    PAssert.that(output).containsInAnyOrder(
+        KV.of(3, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1))),
+        KV.of(42, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1))));
+    pipeline.run();
+  }
+
   @Test
   public void testWithOutputTagsDisplayData() {
     DoFn<String, String> fn = new DoFn<String, String>() {

From 7ece1647d6aeab4544d994a3a00360919920a978 Mon Sep 17 00:00:00 2001
From: JingsongLi <lzljs3620320@aliyun.com>
Date: Tue, 2 May 2017 17:36:46 +0800
Subject: [PATCH 041/387] Fix initial watermark of DoFnOperator in Flink runner

---
 .../wrappers/streaming/DoFnOperator.java      | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index 16bf5d253419..518d6bed308d 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -198,8 +198,8 @@ protected DoFn<InputT, FnOutputT> getDoFn() {
   public void open() throws Exception {
     super.open();
 
-    currentInputWatermark = Long.MIN_VALUE;
-    currentOutputWatermark = Long.MIN_VALUE;
+    setCurrentInputWatermark(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis());
+    setCurrentOutputWatermark(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis());
 
     sideInputReader = NullSideInputReader.of(sideInputs);
 
@@ -429,18 +429,18 @@ public void processWatermark(Watermark mark) throws Exception {
   @Override
   public void processWatermark1(Watermark mark) throws Exception {
     if (keyCoder == null) {
-      this.currentInputWatermark = mark.getTimestamp();
+      setCurrentInputWatermark(mark.getTimestamp());
       long potentialOutputWatermark =
           Math.min(getPushbackWatermarkHold(), currentInputWatermark);
       if (potentialOutputWatermark > currentOutputWatermark) {
-        currentOutputWatermark = potentialOutputWatermark;
+        setCurrentOutputWatermark(potentialOutputWatermark);
         output.emitWatermark(new Watermark(currentOutputWatermark));
       }
     } else {
       // fireTimers, so we need startBundle.
       pushbackDoFnRunner.startBundle();
 
-      this.currentInputWatermark = mark.getTimestamp();
+      setCurrentInputWatermark(mark.getTimestamp());
 
       // hold back by the pushed back values waiting for side inputs
       long actualInputWatermark = Math.min(getPushbackWatermarkHold(), mark.getTimestamp());
@@ -454,7 +454,7 @@ public void processWatermark1(Watermark mark) throws Exception {
       long potentialOutputWatermark = Math.min(currentInputWatermark, combinedWatermarkHold);
 
       if (potentialOutputWatermark > currentOutputWatermark) {
-        currentOutputWatermark = potentialOutputWatermark;
+        setCurrentOutputWatermark(potentialOutputWatermark);
         output.emitWatermark(new Watermark(currentOutputWatermark));
       }
       pushbackDoFnRunner.finishBundle();
@@ -608,6 +608,14 @@ public void fireTimer(InternalTimer<?, TimerData> timer) {
         timerData.getTimestamp(), timerData.getDomain());
   }
 
+  private void setCurrentInputWatermark(long currentInputWatermark) {
+    this.currentInputWatermark = currentInputWatermark;
+  }
+
+  private void setCurrentOutputWatermark(long currentOutputWatermark) {
+    this.currentOutputWatermark = currentOutputWatermark;
+  }
+
   /**
    * Factory for creating an {@link DoFnRunners.OutputManager} from
    * a Flink {@link Output}.

From 78ae52870dd906c63cc3574a6ec0e8e8fa8e510e Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 4 May 2017 12:27:44 -0700
Subject: [PATCH 042/387] [BEAM-59] Remove IOChannelUtils, IOChannelFactory

Leaving one deprecated function in one deprecated class
to enable migration of DataflowRunner.
---
 .../beam/sdk/util/FileIOChannelFactory.java   | 223 ---------------
 .../util/FileIOChannelFactoryRegistrar.java   |  38 ---
 .../beam/sdk/util/IOChannelFactory.java       | 127 ---------
 .../sdk/util/IOChannelFactoryRegistrar.java   |  48 ----
 .../apache/beam/sdk/util/IOChannelUtils.java  | 259 +-----------------
 .../FileIOChannelFactoryRegistrarTest.java    |  44 ---
 .../sdk/util/FileIOChannelFactoryTest.java    | 242 ----------------
 .../beam/sdk/util/IOChannelUtilsTest.java     | 118 --------
 .../beam/sdk/util/GcsIOChannelFactory.java    | 111 --------
 .../util/GcsIOChannelFactoryRegistrar.java    |  38 ---
 .../GcsIOChannelFactoryRegistrarTest.java     |  44 ---
 .../sdk/util/GcsIOChannelFactoryTest.java     |  43 ---
 12 files changed, 3 insertions(+), 1332 deletions(-)
 delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactory.java
 delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrar.java
 delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactory.java
 delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactoryRegistrar.java
 delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrarTest.java
 delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryTest.java
 delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/IOChannelUtilsTest.java
 delete mode 100644 sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java
 delete mode 100644 sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrar.java
 delete mode 100644 sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrarTest.java
 delete mode 100644 sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactory.java
deleted file mode 100644
index 6f6ba3732602..000000000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactory.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.Path;
-import java.nio.file.PathMatcher;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.regex.Matcher;
-import javax.annotation.Nullable;
-
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Implements {@link IOChannelFactory} for local files.
- */
-public class FileIOChannelFactory implements IOChannelFactory {
-  private static final Logger LOG = LoggerFactory.getLogger(FileIOChannelFactory.class);
-
-   /**
-   * Create a {@link FileIOChannelFactory} with the given {@link PipelineOptions}.
-   */
-  public static FileIOChannelFactory fromOptions(@Nullable PipelineOptions options) {
-    return new FileIOChannelFactory();
-  }
-
-  private FileIOChannelFactory() {}
-
-  /**
-   *  Converts the given file spec to a java {@link File}. If {@code spec} is actually a URI with
-   *  the {@code file} scheme, then this function will ensure that the returned {@link File}
-   *  has the correct path.
-   */
-  private static File specToFile(String spec) {
-    try {
-      // Handle URI.
-      URI uri = URI.create(spec);
-      return Paths.get(uri).toFile();
-    } catch (IllegalArgumentException e) {
-      // Fall back to assuming this is actually a file.
-      return Paths.get(spec).toFile();
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * <p>Wildcards in the directory portion are not supported.
-   */
-  @Override
-  public Collection<String> match(String spec) throws IOException {
-    File file = specToFile(spec);
-
-    File parent = file.getAbsoluteFile().getParentFile();
-    if (!parent.exists()) {
-      return Collections.EMPTY_LIST;
-    }
-
-    // Method getAbsolutePath() on Windows platform may return something like
-    // "c:\temp\file.txt". FileSystem.getPathMatcher() call below will treat
-    // '\' (backslash) as an escape character, instead of a directory
-    // separator. Replacing backslash with double-backslash solves the problem.
-    // We perform the replacement on all platforms, even those that allow
-    // backslash as a part of the filename, because Globs.toRegexPattern will
-    // eat one backslash.
-    String pathToMatch = file.getAbsolutePath().replaceAll(Matcher.quoteReplacement("\\"),
-                                                           Matcher.quoteReplacement("\\\\"));
-
-    final PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + pathToMatch);
-
-    Iterable<File> files = com.google.common.io.Files.fileTreeTraverser().preOrderTraversal(parent);
-    Iterable<File> matchedFiles = Iterables.filter(files,
-        Predicates.and(
-            com.google.common.io.Files.isFile(),
-            new Predicate<File>() {
-              @Override
-              public boolean apply(File input) {
-                return matcher.matches(input.toPath());
-              }
-        }));
-
-    List<String> result = new LinkedList<>();
-    for (File match : matchedFiles) {
-      result.add(match.getPath());
-    }
-
-    return result;
-  }
-
-  @Override
-  public ReadableByteChannel open(String spec) throws IOException {
-    LOG.debug("opening file {}", spec);
-    @SuppressWarnings("resource") // The caller is responsible for closing the channel.
-    FileInputStream inputStream = new FileInputStream(specToFile(spec));
-    // Use this method for creating the channel (rather than new FileChannel) so that we get
-    // regular FileNotFoundException. Closing the underyling channel will close the inputStream.
-    return inputStream.getChannel();
-  }
-
-  @Override
-  public WritableByteChannel create(String spec, String mimeType)
-      throws IOException {
-    LOG.debug("creating file {}", spec);
-    File file = specToFile(spec);
-    if (file.getAbsoluteFile().getParentFile() != null
-        && !file.getAbsoluteFile().getParentFile().exists()
-        && !file.getAbsoluteFile().getParentFile().mkdirs()
-        && !file.getAbsoluteFile().getParentFile().exists()) {
-      throw new IOException("Unable to create parent directories for '" + spec + "'");
-    }
-    return Channels.newChannel(
-        new BufferedOutputStream(new FileOutputStream(file)));
-  }
-
-  @Override
-  public long getSizeBytes(String spec) throws IOException {
-    try {
-      return Files.size(specToFile(spec).toPath());
-    } catch (NoSuchFileException e) {
-      throw new FileNotFoundException(e.getReason());
-    }
-  }
-
-  @Override
-  public boolean isReadSeekEfficient(String spec) throws IOException {
-    return true;
-  }
-
-  @Override
-  public String resolve(String path, String other) throws IOException {
-    return toPath(path).resolve(other).toString();
-  }
-
-  @Override
-  public Path toPath(String path) {
-    return specToFile(path).toPath();
-  }
-
-  @Override
-  public void copy(Iterable<String> srcFilenames, Iterable<String> destFilenames) throws
-      IOException {
-    List<String> srcList = Lists.newArrayList(srcFilenames);
-    List<String> destList = Lists.newArrayList(destFilenames);
-    checkArgument(
-        srcList.size() == destList.size(),
-        "Number of source files %s must equal number of destination files %s",
-        srcList.size(),
-        destList.size());
-    int numFiles = srcList.size();
-    for (int i = 0; i < numFiles; i++) {
-      String src = srcList.get(i);
-      String dst = destList.get(i);
-      LOG.info("Copying {} to {}", src, dst);
-      try {
-        // Copy the source file, replacing the existing destination.
-        // Paths.get(x) will not work on Windows OSes cause of the ":" after the drive letter.
-        Files.copy(
-            new File(src).toPath(),
-            new File(dst).toPath(),
-            StandardCopyOption.REPLACE_EXISTING);
-      } catch (NoSuchFileException e) {
-        LOG.info("{} does not exist.", src);
-        // Suppress exception if file does not exist.
-      }
-    }
-  }
-
-  @Override
-  public void remove(Collection<String> filesOrDirs) throws IOException {
-    for (String fileOrDir : filesOrDirs) {
-      LOG.debug("Removing file {}", fileOrDir);
-      removeOne(fileOrDir);
-    }
-  }
-
-  private void removeOne(String fileOrDir) throws IOException {
-    // Delete the file if it exists.
-    boolean exists = Files.deleteIfExists(Paths.get(fileOrDir));
-    if (!exists) {
-      LOG.debug("Tried to delete {}, but it did not exist", fileOrDir);
-    }
-  }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrar.java
deleted file mode 100644
index acc0222832f2..000000000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrar.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import com.google.auto.service.AutoService;
-import org.apache.beam.sdk.options.PipelineOptions;
-
-/**
- * {@link AutoService} registrar for the {@link FileIOChannelFactory}.
- */
-@AutoService(IOChannelFactoryRegistrar.class)
-public class FileIOChannelFactoryRegistrar implements IOChannelFactoryRegistrar {
-
-  @Override
-  public IOChannelFactory fromOptions(PipelineOptions options) {
-    return FileIOChannelFactory.fromOptions(options);
-  }
-
-  @Override
-  public String getScheme() {
-    return "file";
-  }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactory.java
deleted file mode 100644
index 3a3af170fbca..000000000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactory.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.nio.file.Path;
-import java.util.Collection;
-
-/**
- * Defines a factory for working with read and write channels.
- *
- * <p>Channels provide an abstract API for IO operations.
- *
- * <p>See <a href="http://docs.oracle.com/javase/7/docs/api/java/nio/channels/package-summary.html"
- * >Java NIO Channels</a>
- *
- * @deprecated This is under redesign, see: https://issues.apache.org/jira/browse/BEAM-59.
- */
-@Deprecated
-public interface IOChannelFactory {
-
-  /**
-   * Matches a specification, which may contain globs, against available
-   * resources.
-   *
-   * <p>Glob handling is dependent on the implementation.  Implementations should
-   * all support globs in the final component of a path (eg /foo/bar/*.txt),
-   * however they are not required to support globs in the directory paths.
-   *
-   * <p>The list of resources returned are required to exist and not represent abstract
-   * resources such as symlinks and directories.
-   */
-  Collection<String> match(String spec) throws IOException;
-
-  /**
-   * Returns a read channel for the given specification.
-   *
-   * <p>The specification is not expanded; it is used verbatim.
-   *
-   * <p>If seeking is supported, then this returns a
-   * {@link java.nio.channels.SeekableByteChannel}.
-   */
-  ReadableByteChannel open(String spec) throws IOException;
-
-  /**
-   * Returns a write channel for the given specification.
-   *
-   * <p>The specification is not expanded; is it used verbatim.
-   */
-  WritableByteChannel create(String spec, String mimeType) throws IOException;
-
-  /**
-   * Returns the size in bytes for the given specification.
-   *
-   * <p>The specification is not expanded; it is used verbatim.
-   *
-   * <p>{@link FileNotFoundException} will be thrown if the resource does not exist.
-   */
-  long getSizeBytes(String spec) throws IOException;
-
-  /**
-   * Returns {@code true} if the channel created when invoking method {@link #open} for the given
-   * file specification is guaranteed to be of type {@link java.nio.channels.SeekableByteChannel
-   * SeekableByteChannel} and if seeking into positions of the channel is recommended. Returns
-   * {@code false} if the channel returned is not a {@code SeekableByteChannel}. May return
-   * {@code false} even if the channel returned is a {@code SeekableByteChannel}, if seeking is not
-   * efficient for the given file specification.
-   *
-   * <p>Only efficiently seekable files can be split into offset ranges.
-   *
-   * <p>The specification is not expanded; it is used verbatim.
-   */
-  boolean isReadSeekEfficient(String spec) throws IOException;
-
-  /**
-   * Resolve the given {@code other} against the {@code path}.
-   *
-   * <p>If the {@code other} parameter is an absolute path then this method trivially returns
-   * other. If {@code other} is an empty path then this method trivially returns the given
-   * {@code path}. Otherwise this method considers the given {@code path} to be a directory and
-   * resolves the {@code other} path against this path. In the simplest case, the {@code other}
-   * path does not have a root component, in which case this method joins the {@code other} path
-   * to the given {@code path} and returns a resulting path that ends with the {@code other} path.
-   * Where the {@code other} path has a root component then resolution is highly implementation
-   * dependent and therefore unspecified.
-   */
-  String resolve(String path, String other) throws IOException;
-
-  /** Converts the given string to a {@link Path}. */
-  Path toPath(String path);
-
-  /**
-   * Copies a collection of files from one location to another.
-   *
-   * <p>The number of source filenames must equal the number of destination filenames.
-   *
-   * @param srcFilenames the source filenames.
-   * @param destFilenames the destination filenames.
-   */
-  void copy(Iterable<String> srcFilenames, Iterable<String> destFilenames) throws IOException;
-
-  /**
-   * Removes a collection of files or directories.
-   *
-   * <p>Directories are required to be empty. Non-empty directories will not be deleted,
-   * and this method may return silently or throw an exception.
-   */
-  void remove(Collection<String> filesOrDirs) throws IOException;
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactoryRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactoryRegistrar.java
deleted file mode 100644
index 7776b130b771..000000000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelFactoryRegistrar.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import com.google.auto.service.AutoService;
-import java.util.ServiceLoader;
-import org.apache.beam.sdk.options.PipelineOptions;
-
-/**
- * A registrar that creates {@link IOChannelFactory} instances from {@link PipelineOptions}.
- *
- * <p>{@link IOChannelFactory} creators have the ability to provide a registrar by creating
- * a {@link ServiceLoader} entry and a concrete implementation of this interface.
- *
- * <p>It is optional but recommended to use one of the many build time tools such as
- * {@link AutoService} to generate the necessary META-INF files automatically.
- */
-public interface IOChannelFactoryRegistrar {
-  /**
-   * Create a {@link IOChannelFactory} from the given {@link PipelineOptions}.
-   */
-  IOChannelFactory fromOptions(PipelineOptions options);
-
-  /**
-   * Get the URI scheme which defines the namespace of the IOChannelFactoryRegistrar.
-   *
-   * <p>The scheme is required to be unique among all
-   * {@link IOChannelFactoryRegistrar IOChannelFactoryRegistrars}.
-   *
-   * @see <a href="https://www.ietf.org/rfc/rfc2396.txt">RFC 2396</a>
-   */
-  String getScheme();
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java
index 33913f85b9c8..b658983eea87 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java
@@ -17,264 +17,11 @@
  */
 package org.apache.beam.sdk.util;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.Sets;
-import com.google.common.collect.TreeMultimap;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.ServiceLoader;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import javax.annotation.Nonnull;
-import org.apache.beam.sdk.io.DefaultFilenamePolicy;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.common.ReflectHelpers;
 
-/**
- * Provides utilities for creating read and write channels.
- */
+/** Do not use, being removed. */
+@Deprecated
 public class IOChannelUtils {
-  // TODO: add registration mechanism for adding new schemas.
-  private static final Map<String, IOChannelFactory> FACTORY_MAP =
-      Collections.synchronizedMap(new HashMap<String, IOChannelFactory>());
-
-  private static final ClassLoader CLASS_LOADER = ReflectHelpers.findClassLoader();
-
-  /**
-   * Associates a scheme with an {@link IOChannelFactory}.
-   *
-   * <p>The given factory is used to construct read and write channels when
-   * a URI is provided with the given scheme.
-   *
-   * <p>For example, when reading from "gs://bucket/path", the scheme "gs" is
-   * used to lookup the appropriate factory.
-   *
-   * <p>{@link PipelineOptions} are required to provide dependencies and
-   * pipeline level configuration to the individual {@link IOChannelFactory IOChannelFactories}.
-   *
-   * @throws IllegalStateException if multiple {@link IOChannelFactory IOChannelFactories}
-   * for the same scheme are detected.
-   */
-  @VisibleForTesting
-  public static void setIOFactoryInternal(
-      String scheme,
-      IOChannelFactory factory,
-      boolean override) {
-    if (!override && FACTORY_MAP.containsKey(scheme)) {
-      throw new IllegalStateException(String.format(
-          "Failed to register IOChannelFactory: %s. "
-              + "Scheme: [%s] is already registered with %s, and override is not allowed.",
-          FACTORY_MAP.get(scheme).getClass(),
-          scheme,
-          factory.getClass()));
-    }
-    FACTORY_MAP.put(scheme, factory);
-  }
-
-  /**
-   * Deregisters the scheme and the associated {@link IOChannelFactory}.
-   */
-  @VisibleForTesting
-  static void deregisterScheme(String scheme) {
-    FACTORY_MAP.remove(scheme);
-  }
-
-  /**
-   * Registers all {@link IOChannelFactory IOChannelFactories} from {@link ServiceLoader}.
-   *
-   * <p>{@link PipelineOptions} are required to provide dependencies and
-   * pipeline level configuration to the individual {@link IOChannelFactory IOChannelFactories}.
-   *
-   * <p>Multiple {@link IOChannelFactory IOChannelFactories} for the same scheme are not allowed.
-   *
-   * @throws IllegalStateException if multiple {@link IOChannelFactory IOChannelFactories}
-   * for the same scheme are detected.
-   */
-  public static void registerIOFactories(PipelineOptions options) {
-    registerIOFactoriesInternal(options, false /* override */);
-  }
-
-  /**
-   * Registers all {@link IOChannelFactory IOChannelFactories} from {@link ServiceLoader}.
-   *
-   * <p>This requires {@link PipelineOptions} to provide, e.g., credentials for GCS.
-   *
-   * <p>Override existing schemes is allowed.
-   *
-   * @deprecated This is currently to provide different configurations for tests and
-   * is still public for IOChannelFactory redesign purposes.
-   */
   @Deprecated
-  @VisibleForTesting
-  public static void registerIOFactoriesAllowOverride(PipelineOptions options) {
-    registerIOFactoriesInternal(options, true /* override */);
-  }
-
-  private static void registerIOFactoriesInternal(
-      PipelineOptions options, boolean override) {
-    Set<IOChannelFactoryRegistrar> registrars =
-        Sets.newTreeSet(ReflectHelpers.ObjectsClassComparator.INSTANCE);
-    registrars.addAll(Lists.newArrayList(
-        ServiceLoader.load(IOChannelFactoryRegistrar.class, CLASS_LOADER)));
-
-    checkDuplicateScheme(registrars);
-
-    for (IOChannelFactoryRegistrar registrar : registrars) {
-      setIOFactoryInternal(
-          registrar.getScheme(),
-          registrar.fromOptions(options),
-          override);
-    }
-  }
-
-  @VisibleForTesting
-  static void checkDuplicateScheme(Set<IOChannelFactoryRegistrar> registrars) {
-    Multimap<String, IOChannelFactoryRegistrar> registrarsBySchemes =
-        TreeMultimap.create(Ordering.<String>natural(), Ordering.arbitrary());
-
-    for (IOChannelFactoryRegistrar registrar : registrars) {
-      registrarsBySchemes.put(registrar.getScheme(), registrar);
-    }
-    for (Entry<String, Collection<IOChannelFactoryRegistrar>> entry
-        : registrarsBySchemes.asMap().entrySet()) {
-      if (entry.getValue().size() > 1) {
-        String conflictingRegistrars = Joiner.on(", ").join(
-            FluentIterable.from(entry.getValue())
-                .transform(new Function<IOChannelFactoryRegistrar, String>() {
-                  @Override
-                  public String apply(@Nonnull IOChannelFactoryRegistrar input) {
-                    return input.getClass().getName();
-                  }})
-                .toSortedList(Ordering.<String>natural()));
-        throw new IllegalStateException(String.format(
-            "Scheme: [%s] has conflicting registrars: [%s]",
-            entry.getKey(),
-            conflictingRegistrars));
-      }
-    }
-  }
-
-  /**
-   * Creates a write channel for the given filename.
-   */
-  public static WritableByteChannel create(String filename, String mimeType)
-      throws IOException {
-    return getFactory(filename).create(filename, mimeType);
-  }
-
-  /**
-   * Creates a read channel for the given filename.
-   */
-  public static ReadableByteChannel open(String filename)
-      throws IOException {
-    return getFactory(filename).open(filename);
-  }
-
-  /**
-   * Creates a write channel for the given file components.
-   *
-   * <p>If numShards is specified, then a ShardingWritableByteChannel is
-   * returned.
-   *
-   * <p>Shard numbers are 0 based, meaning they start with 0 and end at the
-   * number of shards - 1.
-   */
-  public static WritableByteChannel create(String prefix, String shardTemplate,
-      String suffix, int numShards, String mimeType) throws IOException {
-    if (numShards == 1) {
-      return create(DefaultFilenamePolicy.constructName(prefix, shardTemplate, suffix, 0, 1),
-                    mimeType);
-    }
-
-    // It is the callers responsibility to close this channel.
-    @SuppressWarnings("resource")
-    ShardingWritableByteChannel shardingChannel =
-        new ShardingWritableByteChannel();
-
-    Set<String> outputNames = new HashSet<>();
-    for (int i = 0; i < numShards; i++) {
-      String outputName =
-          DefaultFilenamePolicy.constructName(prefix, shardTemplate, suffix, i, numShards);
-      if (!outputNames.add(outputName)) {
-        throw new IllegalArgumentException(
-            "Shard name collision detected for: " + outputName);
-      }
-      WritableByteChannel channel = create(outputName, mimeType);
-      shardingChannel.addChannel(channel);
-    }
-
-    return shardingChannel;
-  }
-
-  /**
-   * Returns the size in bytes for the given specification.
-   *
-   * <p>The specification is not expanded; it is used verbatim.
-   *
-   * <p>{@link FileNotFoundException} will be thrown if the resource does not exist.
-   */
-  public static long getSizeBytes(String spec) throws IOException {
-    return getFactory(spec).getSizeBytes(spec);
-  }
-
-  private static final Pattern URI_SCHEME_PATTERN = Pattern.compile(
-      "(?<scheme>[a-zA-Z][-a-zA-Z0-9+.]*)://.*");
-
-  /**
-   * Returns the IOChannelFactory associated with an input specification.
-   */
-  public static IOChannelFactory getFactory(String spec) throws IOException {
-    // The spec is almost, but not quite, a URI. In particular,
-    // the reserved characters '[', ']', and '?' have meanings that differ
-    // from their use in the URI spec. ('*' is not reserved).
-    // Here, we just need the scheme, which is so circumscribed as to be
-    // very easy to extract with a regex.
-    Matcher matcher = URI_SCHEME_PATTERN.matcher(spec);
-
-    if (!matcher.matches()) {
-      return FileIOChannelFactory.fromOptions(null);
-    }
-
-    String scheme = matcher.group("scheme");
-    IOChannelFactory ioFactory = FACTORY_MAP.get(scheme);
-    if (ioFactory != null) {
-      return ioFactory;
-    }
-
-    throw new IOException("Unable to find handler for " + spec);
-  }
-
-  /**
-   * Resolve multiple {@code others} against the {@code path} sequentially.
-   *
-   * <p>Empty paths in {@code others} are ignored. If {@code others} contains one or more
-   * absolute paths, then this method returns a path that starts with the last absolute path
-   * in {@code others} joined with the remaining paths. Resolution of paths is highly
-   * implementation dependent and therefore unspecified.
-   */
-  public static String resolve(String path, String... others) throws IOException {
-    IOChannelFactory ioFactory = getFactory(path);
-    String fullPath = path;
-
-    for (String other : others) {
-      fullPath = ioFactory.resolve(fullPath, other);
-    }
-
-    return fullPath;
-  }
+  public static void registerIOFactoriesAllowOverride(PipelineOptions options) {}
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrarTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrarTest.java
deleted file mode 100644
index f8f53e7a51fc..000000000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryRegistrarTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static org.junit.Assert.fail;
-
-import com.google.common.collect.Lists;
-import java.util.ServiceLoader;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link FileIOChannelFactoryRegistrar}.
- */
-@RunWith(JUnit4.class)
-public class FileIOChannelFactoryRegistrarTest {
-
-  @Test
-  public void testServiceLoader() {
-    for (IOChannelFactoryRegistrar registrar
-        : Lists.newArrayList(ServiceLoader.load(IOChannelFactoryRegistrar.class).iterator())) {
-      if (registrar instanceof FileIOChannelFactoryRegistrar) {
-        return;
-      }
-    }
-    fail("Expected to find " + FileIOChannelFactoryRegistrar.class);
-  }
-}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryTest.java
deleted file mode 100644
index 60626195ebad..000000000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FileIOChannelFactoryTest.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.Files;
-import com.google.common.io.LineReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.Reader;
-import java.io.Writer;
-import java.nio.channels.Channels;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.util.List;
-import org.hamcrest.Matchers;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link FileIOChannelFactory}. */
-@RunWith(JUnit4.class)
-public class FileIOChannelFactoryTest {
-  @Rule public ExpectedException thrown = ExpectedException.none();
-  @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
-  private FileIOChannelFactory factory = FileIOChannelFactory.fromOptions(null);
-
-  private void testCreate(Path path) throws Exception {
-    String expected = "my test string";
-    // First with the path string
-    try (Writer writer = Channels.newWriter(
-        factory.create(path.toString(), MimeTypes.TEXT), StandardCharsets.UTF_8.name())) {
-      writer.write(expected);
-    }
-    assertThat(
-        Files.readLines(path.toFile(), StandardCharsets.UTF_8),
-        containsInAnyOrder(expected));
-
-    // Delete the file before trying as URI
-    assertTrue("Unable to delete file " + path, path.toFile().delete());
-
-    // Second with the path URI
-    try (Writer writer = Channels.newWriter(
-        factory.create(path.toUri().toString(), MimeTypes.TEXT), StandardCharsets.UTF_8.name())) {
-      writer.write(expected);
-    }
-    assertThat(
-        Files.readLines(path.toFile(), StandardCharsets.UTF_8),
-        containsInAnyOrder(expected));
-  }
-
-  @Test
-  public void testCreateWithExistingFile() throws Exception {
-    File existingFile = temporaryFolder.newFile();
-    testCreate(existingFile.toPath());
-  }
-
-  @Test
-  public void testCreateWithinExistingDirectory() throws Exception {
-    testCreate(temporaryFolder.getRoot().toPath().resolve("file.txt"));
-  }
-
-  @Test
-  public void testCreateWithNonExistentSubDirectory() throws Exception {
-    testCreate(temporaryFolder.getRoot().toPath().resolve("non-existent-dir").resolve("file.txt"));
-  }
-
-  @Test
-  public void testReadWithExistingFile() throws Exception {
-    String expected = "my test string";
-    File existingFile = temporaryFolder.newFile();
-    Files.write(expected, existingFile, StandardCharsets.UTF_8);
-    String data;
-    try (Reader reader =
-        Channels.newReader(factory.open(existingFile.getPath()), StandardCharsets.UTF_8.name())) {
-      data = new LineReader(reader).readLine();
-    }
-    assertEquals(expected, data);
-  }
-
-  @Test
-  public void testReadNonExistentFile() throws Exception {
-    thrown.expect(FileNotFoundException.class);
-    factory
-        .open(
-            temporaryFolder
-                .getRoot()
-                .toPath()
-                .resolve("non-existent-file.txt")
-                .toString())
-        .close();
-  }
-
-  @Test
-  public void testIsReadSeekEfficient() throws Exception {
-    assertTrue(factory.isReadSeekEfficient("somePath"));
-  }
-
-  @Test
-  public void testMatchExact() throws Exception {
-    List<String> expected = ImmutableList.of(temporaryFolder.newFile("a").toString());
-    temporaryFolder.newFile("aa");
-    temporaryFolder.newFile("ab");
-
-    assertThat(factory.match(temporaryFolder.getRoot().toPath().resolve("a").toString()),
-        containsInAnyOrder(expected.toArray(new String[expected.size()])));
-  }
-
-  @Test
-  public void testMatchPatternNone() throws Exception {
-    List<String> expected = ImmutableList.of();
-    temporaryFolder.newFile("a");
-    temporaryFolder.newFile("aa");
-    temporaryFolder.newFile("ab");
-
-    // Windows doesn't like resolving paths with * in them, so the * is appended after resolve.
-    assertThat(factory.match(factory.resolve(temporaryFolder.getRoot().getPath(), "b") + "*"),
-        containsInAnyOrder(expected.toArray(new String[expected.size()])));
-  }
-
-  @Test
-  public void testMatchForNonExistentFile() throws Exception {
-    List<String> expected = ImmutableList.of();
-    temporaryFolder.newFile("aa");
-
-    assertThat(factory.match(factory.resolve(temporaryFolder.getRoot().getPath(), "a")),
-        containsInAnyOrder(expected.toArray(new String[expected.size()])));
-  }
-
-  @Test
-  public void testMatchMultipleWithoutSubdirectoryExpansion() throws Exception {
-    File unmatchedSubDir = temporaryFolder.newFolder("aaa");
-    File unmatchedSubDirFile = File.createTempFile("sub-dir-file", "", unmatchedSubDir);
-    unmatchedSubDirFile.deleteOnExit();
-    List<String> expected = ImmutableList.of(temporaryFolder.newFile("a").toString(),
-        temporaryFolder.newFile("aa").toString(), temporaryFolder.newFile("ab").toString());
-    temporaryFolder.newFile("ba");
-    temporaryFolder.newFile("bb");
-
-    // Windows doesn't like resolving paths with * in them, so the * is appended after resolve.
-    assertThat(factory.match(factory.resolve(temporaryFolder.getRoot().getPath(), "a") + "*"),
-        containsInAnyOrder(expected.toArray(new String[expected.size()])));
-  }
-
-  @Test
-  public void testMatchMultipleWithSubdirectoryExpansion() throws Exception {
-    File matchedSubDir = temporaryFolder.newFolder("a");
-    File matchedSubDirFile = File.createTempFile("sub-dir-file", "", matchedSubDir);
-    matchedSubDirFile.deleteOnExit();
-    File unmatchedSubDir = temporaryFolder.newFolder("b");
-    File unmatchedSubDirFile = File.createTempFile("sub-dir-file", "", unmatchedSubDir);
-    unmatchedSubDirFile.deleteOnExit();
-
-    List<String> expected = ImmutableList.of(matchedSubDirFile.toString(),
-        temporaryFolder.newFile("aa").toString(), temporaryFolder.newFile("ab").toString());
-    temporaryFolder.newFile("ba");
-    temporaryFolder.newFile("bb");
-
-    // Windows doesn't like resolving paths with * in them, so the ** is appended after resolve.
-    assertThat(factory.match(factory.resolve(temporaryFolder.getRoot().getPath(), "a") + "**"),
-        Matchers.hasItems(expected.toArray(new String[expected.size()])));
-  }
-
-  @Test
-  public void testMatchWithDirectoryFiltersOutDirectory() throws Exception {
-    List<String> expected = ImmutableList.of(temporaryFolder.newFile("a").toString());
-    temporaryFolder.newFolder("a_dir_that_should_not_be_matched");
-
-    // Windows doesn't like resolving paths with * in them, so the * is appended after resolve.
-    assertThat(factory.match(factory.resolve(temporaryFolder.getRoot().getPath(), "a") + "*"),
-        containsInAnyOrder(expected.toArray(new String[expected.size()])));
-  }
-
-  @Test
-  public void testMatchWithoutParentDirectory() throws Exception {
-    String pattern = factory.resolve(
-        factory.resolve(temporaryFolder.getRoot().getPath(), "non_existing_dir"),
-        "*");
-    assertTrue(factory.match(pattern).isEmpty());
-  }
-
-  @Test
-  public void testResolve() throws Exception {
-    Path rootPath = temporaryFolder.getRoot().toPath();
-    String rootString = rootPath.toString();
-
-    String expected = rootPath.resolve("aa").toString();
-    assertEquals(expected, factory.resolve(rootString, "aa"));
-    assertEquals(expected, factory.resolve("file:" + rootString, "aa"));
-    assertEquals(expected, factory.resolve("file://" + rootString, "aa"));
-  }
-
-  @Test
-  public void testResolveOtherIsFullPath() throws Exception {
-    String expected = temporaryFolder.getRoot().getPath();
-    assertEquals(expected, factory.resolve(expected, expected));
-  }
-
-  @Test
-  public void testResolveOtherIsEmptyPath() throws Exception {
-    String expected = temporaryFolder.getRoot().getPath();
-    assertEquals(expected, factory.resolve(expected, ""));
-  }
-
-  @Test
-  public void testGetSizeBytes() throws Exception {
-    String data = "TestData!!!";
-    File file = temporaryFolder.newFile();
-    Files.write(data, file, StandardCharsets.UTF_8);
-    assertEquals(data.length(), factory.getSizeBytes(file.getPath()));
-  }
-
-  @Test
-  public void testGetSizeBytesForNonExistentFile() throws Exception {
-    thrown.expect(FileNotFoundException.class);
-    factory.getSizeBytes(
-        factory.resolve(temporaryFolder.getRoot().getPath(), "non-existent-file"));
-  }
-}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IOChannelUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IOChannelUtilsTest.java
deleted file mode 100644
index ea4ae87ef075..000000000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IOChannelUtilsTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static org.hamcrest.Matchers.instanceOf;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-
-import com.google.common.collect.Sets;
-import com.google.common.io.Files;
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for IOChannelUtils.
- */
-@RunWith(JUnit4.class)
-public class IOChannelUtilsTest {
-  @Rule
-  public TemporaryFolder tmpFolder = new TemporaryFolder();
-
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  @Test(expected = IllegalArgumentException.class)
-  public void testShardNameCollision() throws Exception {
-    File outFolder = tmpFolder.newFolder();
-    String filename = outFolder.toPath().resolve("output").toString();
-
-    IOChannelUtils.create(filename, "", "", 2, "text").close();
-    fail("IOChannelUtils.create expected to fail due "
-        + "to filename collision");
-  }
-
-  @Test
-  public void testHandlerNoScheme() throws Exception {
-    String pathToTempFolder = tmpFolder.getRoot().getAbsolutePath();
-    assertThat(IOChannelUtils.getFactory(pathToTempFolder), instanceOf(FileIOChannelFactory.class));
-  }
-
-  @Test
-  public void testGetSizeBytes() throws Exception {
-    String data = "TestData";
-    File file = tmpFolder.newFile();
-    Files.write(data, file, StandardCharsets.UTF_8);
-    assertEquals(data.length(), IOChannelUtils.getSizeBytes(file.getPath()));
-  }
-
-  @Test
-  public void testResolveSinglePath() throws Exception {
-    String expected = tmpFolder.getRoot().toPath().resolve("aa").toString();
-    assertEquals(expected, IOChannelUtils.resolve(tmpFolder.getRoot().toString(), "aa"));
-  }
-
-  @Test
-  public void testResolveMultiplePaths() throws Exception {
-    String expected =
-        tmpFolder.getRoot().toPath().resolve("aa").resolve("bb").resolve("cc").toString();
-    assertEquals(expected,
-        IOChannelUtils.resolve(tmpFolder.getRoot().getPath(), "aa", "bb", "cc"));
-  }
-
-  @Test
-  public void testRegisterIOFactoriesAllowOverride() throws Exception {
-    IOChannelUtils.registerIOFactoriesAllowOverride(PipelineOptionsFactory.create());
-    IOChannelUtils.registerIOFactoriesAllowOverride(PipelineOptionsFactory.create());
-    assertNotNull(IOChannelUtils.getFactory("gs"));
-    assertNotNull(IOChannelUtils.getFactory("file"));
-  }
-
-  @Test
-  public void testRegisterIOFactories() throws Exception {
-    IOChannelUtils.deregisterScheme("gs");
-    IOChannelUtils.deregisterScheme("file");
-
-    IOChannelUtils.registerIOFactories(PipelineOptionsFactory.create());
-    assertNotNull(IOChannelUtils.getFactory("gs"));
-    assertNotNull(IOChannelUtils.getFactory("file"));
-    thrown.expect(RuntimeException.class);
-    thrown.expectMessage("Failed to register IOChannelFactory");
-    thrown.expectMessage("override is not allowed");
-    IOChannelUtils.registerIOFactories(PipelineOptionsFactory.create());
-  }
-
-  @Test
-  public void testCheckDuplicateScheme() throws Exception {
-    thrown.expect(RuntimeException.class);
-    thrown.expectMessage("Scheme: [file] has conflicting registrars");
-    IOChannelUtils.checkDuplicateScheme(
-        Sets.<IOChannelFactoryRegistrar>newHashSet(
-            new FileIOChannelFactoryRegistrar(),
-            new FileIOChannelFactoryRegistrar()));
-  }
-}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java
deleted file mode 100644
index 3a12620d0f2c..000000000000
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactory.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.nio.file.Path;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.gcsfs.GcsPath;
-
-/**
- * Implements IOChannelFactory for GCS.
- */
-public class GcsIOChannelFactory implements IOChannelFactory {
-
-  /**
-   * Create a {@link GcsIOChannelFactory} with the given {@link PipelineOptions}.
-   */
-  public static GcsIOChannelFactory fromOptions(PipelineOptions options) {
-    return new GcsIOChannelFactory(options.as(GcsOptions.class));
-  }
-
-  private final GcsOptions options;
-
-  private GcsIOChannelFactory(GcsOptions options) {
-    this.options = options;
-  }
-
-  @Override
-  public Collection<String> match(String spec) throws IOException {
-    GcsPath path = GcsPath.fromUri(spec);
-    GcsUtil util = options.getGcsUtil();
-    List<GcsPath> matched = util.expand(path);
-
-    List<String> specs = new LinkedList<>();
-    for (GcsPath match : matched) {
-      specs.add(match.toString());
-    }
-
-    return specs;
-  }
-
-  @Override
-  public ReadableByteChannel open(String spec) throws IOException {
-    GcsPath path = GcsPath.fromUri(spec);
-    GcsUtil util = options.getGcsUtil();
-    return util.open(path);
-  }
-
-  @Override
-  public WritableByteChannel create(String spec, String mimeType)
-      throws IOException {
-    GcsPath path = GcsPath.fromUri(spec);
-    GcsUtil util = options.getGcsUtil();
-    return util.create(path, mimeType);
-  }
-
-  @Override
-  public long getSizeBytes(String spec) throws IOException {
-    GcsPath path = GcsPath.fromUri(spec);
-    GcsUtil util = options.getGcsUtil();
-    return util.fileSize(path);
-  }
-
-  @Override
-  public boolean isReadSeekEfficient(String spec) throws IOException {
-    // TODO It is incorrect to return true here for files with content encoding set to gzip.
-    return true;
-  }
-
-  @Override
-  public String resolve(String path, String other) throws IOException {
-    return toPath(path).resolve(other).toString();
-  }
-
-  @Override
-  public Path toPath(String path) {
-    return GcsPath.fromUri(path);
-  }
-
-  @Override
-  public void copy(Iterable<String> srcFilenames, Iterable<String> destFilenames)
-      throws IOException {
-    options.getGcsUtil().copy(srcFilenames, destFilenames);
-  }
-
-  @Override
-  public void remove(Collection<String> filesOrDirs) throws IOException {
-    options.getGcsUtil().remove(filesOrDirs);
-  }
-}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrar.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrar.java
deleted file mode 100644
index b4c457fcaeb7..000000000000
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrar.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import com.google.auto.service.AutoService;
-import org.apache.beam.sdk.options.PipelineOptions;
-
-/**
- * {@link AutoService} registrar for the {@link GcsIOChannelFactory}.
- */
-@AutoService(IOChannelFactoryRegistrar.class)
-public class GcsIOChannelFactoryRegistrar implements IOChannelFactoryRegistrar {
-
-  @Override
-  public GcsIOChannelFactory fromOptions(PipelineOptions options) {
-    return GcsIOChannelFactory.fromOptions(options);
-  }
-
-  @Override
-  public String getScheme() {
-    return "gs";
-  }
-}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrarTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrarTest.java
deleted file mode 100644
index a29dd45456e4..000000000000
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryRegistrarTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static org.junit.Assert.fail;
-
-import com.google.common.collect.Lists;
-import java.util.ServiceLoader;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link GcsIOChannelFactoryRegistrar}.
- */
-@RunWith(JUnit4.class)
-public class GcsIOChannelFactoryRegistrarTest {
-
-  @Test
-  public void testServiceLoader() {
-    for (IOChannelFactoryRegistrar registrar
-        : Lists.newArrayList(ServiceLoader.load(IOChannelFactoryRegistrar.class).iterator())) {
-      if (registrar instanceof GcsIOChannelFactoryRegistrar) {
-        return;
-      }
-    }
-    fail("Expected to find " + GcsIOChannelFactoryRegistrar.class);
-  }
-}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java
deleted file mode 100644
index f53490add74e..000000000000
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsIOChannelFactoryTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link GcsIOChannelFactoryTest}. */
-@RunWith(JUnit4.class)
-public class GcsIOChannelFactoryTest {
-  private GcsIOChannelFactory factory;
-
-  @Before
-  public void setUp() {
-    factory = GcsIOChannelFactory.fromOptions(PipelineOptionsFactory.as(GcsOptions.class));
-  }
-
-  @Test
-  public void testResolve() throws Exception {
-    assertEquals("gs://bucket/object", factory.resolve("gs://bucket", "object"));
-  }
-}

From 1d9089c968535090ba77977ddecc6cdfb936655f Mon Sep 17 00:00:00 2001
From: Mark Liu <markliu@google.com>
Date: Thu, 4 May 2017 12:48:55 -0700
Subject: [PATCH 043/387] Move thrown.expect to correct place

---
 .../test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
index 630893147f6f..dbeab04c9731 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
@@ -287,11 +287,11 @@ public void testWritingFailsTableDoesNotExist() throws Exception {
         PCollection<KV<byte[], Iterable<Mutation>>> emptyInput =
                 p.apply(Create.empty(HBaseIO.WRITE_CODER));
 
+        emptyInput.apply("write", HBaseIO.write().withConfiguration(conf).withTableId(table));
+
         // Exception will be thrown by write.validate() when write is applied.
         thrown.expect(IllegalArgumentException.class);
         thrown.expectMessage(String.format("Table %s does not exist", table));
-
-        emptyInput.apply("write", HBaseIO.write().withConfiguration(conf).withTableId(table));
         p.run();
     }
 

From 26073817bdfcd64c26dadcb9ed3940a7fc987c59 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Mon, 17 Apr 2017 13:21:14 -0700
Subject: [PATCH 044/387] Note when checkpoints may be committed.

---
 .../apache/beam/sdk/io/UnboundedSource.java   | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/UnboundedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/UnboundedSource.java
index af6a8ccd9e56..b99d5eefaa29 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/UnboundedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/UnboundedSource.java
@@ -83,9 +83,9 @@ public abstract UnboundedReader<OutputT> createReader(
   /**
    * Returns whether this source requires explicit deduping.
    *
-   * <p>This is needed if the underlying data source can return the same record multiple times,
-   * such a queuing system with a pull-ack model.  Sources where the records read are uniquely
-   * identified by the persisted state in the CheckpointMark do not need this.
+   * <p>This is needed if the underlying data source can return the same record multiple times, such
+   * a queuing system with a pull-ack model. Sources where the records read are uniquely identified
+   * by the persisted state in the CheckpointMark do not need this.
    *
    * <p>Generally, if {@link CheckpointMark#finalizeCheckpoint()} is overridden, this method should
    * return true. Checkpoint finalization is best-effort, and readers can be resumed from a
@@ -233,20 +233,25 @@ public byte[] getCurrentRecordId() throws NoSuchElementException {
      * }
      * }</pre>
      *
-     * <p>All elements read up until this method is called will be processed together as a bundle.
-     * (An element is considered 'read' if it could be returned by a call to {@link #getCurrent}.)
-     * Once the result of processing those elements and the returned checkpoint have been durably
+     * <p>All elements read between the last time this method was called (or since this reader was
+     * created, if this method has not been called on this reader) until this method is called will
+     * be processed together as a bundle. (An element is considered 'read' if it could be returned
+     * by a call to {@link #getCurrent}.)
+     *
+     * <p>Once the result of processing those elements and the returned checkpoint have been durably
      * committed, {@link CheckpointMark#finalizeCheckpoint} will be called at most once at some
      * later point on the returned {@link CheckpointMark} object. Checkpoint finalization is
      * best-effort, and checkpoints may not be finalized. If duplicate elements may be produced if
      * checkpoints are not finalized in a timely manner, {@link UnboundedSource#requiresDeduping()}
      * should be overridden to return true, and {@link UnboundedReader#getCurrentRecordId()} should
-     * be overriden to return unique record IDs.
+     * be overridden to return unique record IDs.
+     *
+     * <p>A checkpoint will be committed to durable storage only if all all previous checkpoints
+     * produced by the same reader have also been committed.
      *
      * <p>The returned object should not be modified.
      *
-     * <p>May be called after {@link #advance} or {@link #start} has returned false, but not before
-     * {@link #start} has been called.
+     * <p>May not be called before {@link #start} has been called.
      */
     public abstract CheckpointMark getCheckpointMark();
 

From 513ef1145aa6c563625ab4154296d5aa4fc6c17c Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 4 May 2017 15:01:24 -0700
Subject: [PATCH 045/387] Activate WindowedWordCountIT on Apex runner

---
 examples/java/pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index d673da27fe29..65625df297b3 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -303,6 +303,7 @@
                 <configuration>
                   <includes>
                     <include>WordCountIT.java</include>
+                    <include>WindowedWordCountIT.java</include>
                   </includes>
                   <parallel>all</parallel>
                   <threadCount>4</threadCount>

From d59d9b74ecfd8bd936c768addab25e714a481ed0 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Mon, 1 May 2017 18:11:18 -0700
Subject: [PATCH 046/387] Split StartBundleContext and FinishBundleContext

These methods have different restrictions on their capabilities. Remove
Context.output(), and add a more precise
FinishBundleContext.output(Element, Timestamp, Window)
---
 .../runners/core/BaseExecutionContext.java    |   9 -
 .../beam/runners/core/DoFnAdapters.java       | 111 +++++++++---
 .../beam/runners/core/ExecutionContext.java   |  10 +-
 ...oundedSplittableProcessElementInvoker.java |  13 +-
 .../beam/runners/core/SimpleDoFnRunner.java   | 161 +++++++++++++++---
 .../beam/runners/core/SplittableParDo.java    |  35 ++--
 .../runners/core/SimpleDoFnRunnerTest.java    |   4 +-
 .../runners/core/SplittableParDoTest.java     |   4 +-
 runners/google-cloud-dataflow-java/pom.xml    |   2 +-
 .../runners/dataflow/BatchViewOverrides.java  |   2 +-
 .../org/apache/beam/sdk/io/WriteFiles.java    |   7 +-
 .../beam/sdk/options/PipelineOptions.java     |   3 +-
 .../apache/beam/sdk/transforms/Combine.java   |   5 +-
 .../org/apache/beam/sdk/transforms/DoFn.java  |  79 ++++++---
 .../beam/sdk/transforms/DoFnTester.java       |  62 ++++---
 .../beam/sdk/transforms/GroupIntoBatches.java |   2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java |   5 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java  |  19 ++-
 .../sdk/transforms/reflect/DoFnInvoker.java   |  19 ++-
 .../sdk/transforms/reflect/DoFnSignature.java |  42 +++--
 .../transforms/reflect/DoFnSignatures.java    |  58 +++++--
 .../sdk/transforms/windowing/PaneInfo.java    |   3 +-
 .../apache/beam/sdk/metrics/MetricsTest.java  |   4 +-
 .../beam/sdk/transforms/DoFnTesterTest.java   |   8 +-
 .../sdk/transforms/ParDoLifecycleTest.java    |  16 +-
 .../apache/beam/sdk/transforms/ParDoTest.java |  75 ++------
 .../sdk/transforms/SplittableDoFnTest.java    |   4 +-
 .../transforms/reflect/DoFnInvokersTest.java  |  22 ++-
 .../reflect/DoFnSignaturesTest.java           |  15 +-
 .../control/ProcessBundleHandlerTest.java     |  17 +-
 .../sdk/io/elasticsearch/ElasticsearchIO.java |  12 +-
 .../sdk/io/gcp/bigquery/StreamingWriteFn.java |   4 +-
 .../sdk/io/gcp/bigquery/TagWithUniqueIds.java |   2 +-
 .../io/gcp/bigquery/WriteBundlesToFiles.java  |  17 +-
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java  |   4 +-
 .../sdk/io/gcp/datastore/DatastoreV1.java     |  10 +-
 .../beam/sdk/io/gcp/pubsub/PubsubIO.java      |   4 +-
 .../io/gcp/pubsub/PubsubUnboundedSink.java    |   4 +-
 .../org/apache/beam/sdk/io/hdfs/Write.java    |   7 +-
 .../org/apache/beam/sdk/io/hbase/HBaseIO.java |   7 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java   |  11 +-
 .../org/apache/beam/sdk/io/jms/JmsIO.java     |   6 +-
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java |   2 +-
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java  |   4 +-
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java |  10 +-
 45 files changed, 603 insertions(+), 317 deletions(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
index 8d682563f692..23d61f89b1db 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
@@ -23,7 +23,6 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.DoFn.Context;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.TupleTag;
@@ -105,17 +104,9 @@ public Collection<? extends T> getAllStepContexts() {
     return Collections.unmodifiableCollection(cachedStepContexts.values());
   }
 
-  /**
-   * Hook for subclasses to implement that will be called whenever
-   * {@link Context#output(Object)} is called.
-   */
   @Override
   public void noteOutput(WindowedValue<?> output) {}
 
-  /**
-   * Hook for subclasses to implement that will be called whenever
-   * {@link Context#output(TupleTag, Object)} is called.
-   */
   @Override
   public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {}
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
index b2069e19d010..fd1772ad2761 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
@@ -18,11 +18,13 @@
 package org.apache.beam.runners.core;
 
 import java.io.IOException;
+import org.apache.beam.runners.core.OldDoFn.Context;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.Context;
+import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
 import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
+import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
@@ -81,12 +83,12 @@ public void setup() throws Exception {
     @Override
     public void startBundle(Context c) throws Exception {
       fn.prepareForProcessing();
-      invoker.invokeStartBundle(new ContextAdapter<>(fn, c));
+      invoker.invokeStartBundle(new StartBundleContextAdapter<>(fn, c));
     }
 
     @Override
     public void finishBundle(Context c) throws Exception {
-      invoker.invokeFinishBundle(new ContextAdapter<>(fn, c));
+      invoker.invokeFinishBundle(new FinishBundleContextAdapter<>(fn, c));
     }
 
     @Override
@@ -128,42 +130,86 @@ private static class WindowDoFnAdapter<InputT, OutputT> extends SimpleDoFnAdapte
 
   /**
    * Wraps an {@link OldDoFn.Context} as a {@link DoFnInvoker.ArgumentProvider} inside a {@link
-   * DoFn.StartBundle} or {@link DoFn.FinishBundle} method, which means the extra context is
-   * unavailable.
+   * DoFn.StartBundle} method, which means the extra context is unavailable.
    */
-  private static class ContextAdapter<InputT, OutputT> extends DoFn<InputT, OutputT>.Context
+  private static class StartBundleContextAdapter<InputT, OutputT>
+      extends DoFn<InputT, OutputT>.StartBundleContext
       implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
     private OldDoFn<InputT, OutputT>.Context context;
 
-    private ContextAdapter(DoFn<InputT, OutputT> fn, OldDoFn<InputT, OutputT>.Context context) {
+    private StartBundleContextAdapter(DoFn<InputT, OutputT> fn, Context context) {
       fn.super();
       this.context = context;
     }
 
-    @Override
     public PipelineOptions getPipelineOptions() {
       return context.getPipelineOptions();
     }
 
     @Override
-    public void output(OutputT output) {
-      context.output(output);
+    public BoundedWindow window() {
+      // The OldDoFn doesn't allow us to ask for these outside processElement, so this
+      // should be unreachable.
+      throw new UnsupportedOperationException(
+          "Can only get the window in processElement; elsewhere there is no defined window.");
     }
 
     @Override
-    public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      context.outputWithTimestamp(output, timestamp);
+    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      return this;
     }
 
     @Override
-    public <T> void output(TupleTag<T> tag, T output) {
-      context.output(tag, output);
+    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Can only get a FinishBundleContext in finishBundle");
     }
 
     @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      context.outputWithTimestamp(tag, output, timestamp);
+    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Can only get a ProcessContext in processElement");
+    }
+
+    @Override
+    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Timers are not supported for OldDoFn");
+    }
+
+    @Override
+    public RestrictionTracker<?> restrictionTracker() {
+      throw new UnsupportedOperationException("This is a non-splittable DoFn");
+    }
+
+    @Override
+    public State state(String stateId) {
+      throw new UnsupportedOperationException("State is not supported by this runner");
+    }
+
+    @Override
+    public Timer timer(String timerId) {
+      throw new UnsupportedOperationException("Timers are not supported by this runner");
+    }
+  }
+
+  /**
+   * Wraps an {@link OldDoFn.Context} as a {@link DoFnInvoker.ArgumentProvider} inside a {@link
+   * DoFn.FinishBundle} method, which means the extra context is unavailable.
+   */
+  private static class FinishBundleContextAdapter<InputT, OutputT>
+      extends DoFn<InputT, OutputT>.FinishBundleContext
+      implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+
+    private OldDoFn<InputT, OutputT>.Context context;
+
+    private FinishBundleContextAdapter(DoFn<InputT, OutputT> fn, Context context) {
+      fn.super();
+      this.context = context;
+    }
+
+    public PipelineOptions getPipelineOptions() {
+      return context.getPipelineOptions();
     }
 
     @Override
@@ -175,7 +221,13 @@ public BoundedWindow window() {
     }
 
     @Override
-    public Context context(DoFn<InputT, OutputT> doFn) {
+    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Can only get a StartBundleContext in startBundle");
+    }
+
+    @Override
+    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
       return this;
     }
 
@@ -205,6 +257,20 @@ public State state(String stateId) {
     public Timer timer(String timerId) {
       throw new UnsupportedOperationException("Timers are not supported by this runner");
     }
+
+    @Override
+    public void output(
+        OutputT output, Instant timestamp, BoundedWindow window) {
+      // Not full fidelity conversion. This should be removed as soon as possible.
+      context.outputWithTimestamp(output, timestamp);
+    }
+
+    @Override
+    public <T> void output(
+        TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+      // Not full fidelity conversion. This should be removed as soon as possible.
+      context.outputWithTimestamp(tag, output, timestamp);
+    }
   }
 
   /**
@@ -278,8 +344,13 @@ public BoundedWindow window() {
     }
 
     @Override
-    public Context context(DoFn<InputT, OutputT> doFn) {
-      return this;
+    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      return null;
+    }
+
+    @Override
+    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+      return null;
     }
 
     @Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
index c0d01aeed3b9..d2fdaac50982 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
@@ -20,7 +20,7 @@
 import java.io.IOException;
 import java.util.Collection;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.DoFn.Context;
+import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.TupleTag;
@@ -42,13 +42,13 @@ public interface ExecutionContext {
 
   /**
    * Hook for subclasses to implement that will be called whenever
-   * {@link Context#output(TupleTag, Object)} is called.
+   * {@link WindowedContext#output(TupleTag, Object)} is called.
    */
   void noteOutput(WindowedValue<?> output);
 
   /**
    * Hook for subclasses to implement that will be called whenever
-   * {@link Context#output(TupleTag, Object)} is called.
+   * {@link WindowedContext#output(TupleTag, Object)} is called.
    */
   void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
 
@@ -69,14 +69,14 @@ public interface StepContext {
 
     /**
      * Hook for subclasses to implement that will be called whenever
-     * {@link org.apache.beam.sdk.transforms.DoFn.Context#output}
+     * {@link WindowedContext#output}
      * is called.
      */
     void noteOutput(WindowedValue<?> output);
 
     /**
      * Hook for subclasses to implement that will be called whenever
-     * {@link org.apache.beam.sdk.transforms.DoFn.Context#output}
+     * {@link WindowedContext#output}
      * is called.
      */
     void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index 35d6eb7e2bd6..a51041ed9a99 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -28,6 +28,8 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
+import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -117,9 +119,16 @@ public BoundedWindow window() {
           }
 
           @Override
-          public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
+          public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
             throw new IllegalStateException(
-                "Should not access context() from @"
+                "Should not access startBundleContext() from @"
+                    + DoFn.ProcessElement.class.getSimpleName());
+          }
+
+          @Override
+          public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+            throw new IllegalStateException(
+                "Should not access finishBundleContext() from @"
                     + DoFn.ProcessElement.class.getSimpleName());
           }
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 7f29a6fff52e..768378daceca 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -33,9 +33,10 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.Context;
+import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
 import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
+import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
@@ -137,9 +138,11 @@ public SimpleDoFnRunner(
 
   @Override
   public void startBundle() {
+    DoFnStartBundleContext<InputT, OutputT> startBundleContext =
+        createStartBundleContext(fn, context);
     // This can contain user code. Wrap it in case it throws an exception.
     try {
-      invoker.invokeStartBundle(context);
+      invoker.invokeStartBundle(startBundleContext);
     } catch (Throwable t) {
       // Exception in user code.
       throw wrapUserCodeException(t);
@@ -200,15 +203,27 @@ private void invokeProcessElement(WindowedValue<InputT> elem) {
 
   @Override
   public void finishBundle() {
+    DoFnFinishBundleContext<InputT, OutputT> finishBundleContext =
+        createFinishBundleContext(fn, context);
     // This can contain user code. Wrap it in case it throws an exception.
     try {
-      invoker.invokeFinishBundle(context);
+      invoker.invokeFinishBundle(finishBundleContext);
     } catch (Throwable t) {
       // Exception in user code.
       throw wrapUserCodeException(t);
     }
   }
 
+  private DoFnStartBundleContext<InputT, OutputT> createStartBundleContext(
+      DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+    return new DoFnStartBundleContext<>(this.fn, this.context);
+  }
+
+  private DoFnFinishBundleContext<InputT, OutputT> createFinishBundleContext(
+      DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+    return new DoFnFinishBundleContext<>(fn, context);
+  }
+
   /** Returns a new {@link DoFn.ProcessContext} for the given element. */
   private DoFnProcessContext<InputT, OutputT> createProcessContext(WindowedValue<InputT> elem) {
     return new DoFnProcessContext<InputT, OutputT>(fn, context, elem, allowedLateness);
@@ -228,8 +243,7 @@ private boolean isSystemDoFn() {
    * @param <InputT> the type of the {@link DoFn} (main) input elements
    * @param <OutputT> the type of the {@link DoFn} (main) output elements
    */
-  private static class DoFnContext<InputT, OutputT> extends DoFn<InputT, OutputT>.Context
-      implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+  private static class DoFnContext<InputT, OutputT> {
     private static final int MAX_SIDE_OUTPUTS = 1000;
 
     final PipelineOptions options;
@@ -255,7 +269,6 @@ public DoFnContext(
         List<TupleTag<?>> additionalOutputTags,
         StepContext stepContext,
         WindowFn<?, ?> windowFn) {
-      fn.super();
       this.options = options;
       this.fn = fn;
       this.sideInputReader = sideInputReader;
@@ -274,7 +287,6 @@ public DoFnContext(
 
     //////////////////////////////////////////////////////////////////////////////
 
-    @Override
     public PipelineOptions getPipelineOptions() {
       return options;
     }
@@ -374,30 +386,95 @@ private <T> void outputWindowedValue(TupleTag<T> tag, WindowedValue<T> windowedE
         stepContext.noteOutput(tag, windowedElem);
       }
     }
+  }
+
+
+  /**
+   * A concrete implementation of {@link DoFn.StartBundleContext}.
+   */
+  private class DoFnStartBundleContext<InputT, OutputT>
+      extends DoFn<InputT, OutputT>.StartBundleContext
+      implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+    private final DoFn<InputT, OutputT> fn;
+    private final DoFnContext<InputT, OutputT> context;
+
+    private DoFnStartBundleContext(DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+      fn.super();
+      this.fn = fn;
+      this.context = context;
+    }
 
-    // Following implementations of output, outputWithTimestamp, and output
-    // are only accessible in DoFn.startBundle and DoFn.finishBundle, and will be shadowed by
-    // ProcessContext's versions in DoFn.processElement.
     @Override
-    public void output(OutputT output) {
-      outputWindowedValue(output, null, null, PaneInfo.NO_FIRING);
+    public PipelineOptions getPipelineOptions() {
+      return context.getPipelineOptions();
     }
 
     @Override
-    public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      outputWindowedValue(output, timestamp, null, PaneInfo.NO_FIRING);
+    public BoundedWindow window() {
+      throw new UnsupportedOperationException(
+          "Cannot access window outside of @ProcessElement and @OnTimer methods.");
     }
 
     @Override
-    public <T> void output(TupleTag<T> tag, T output) {
-      checkNotNull(tag, "TupleTag passed to output cannot be null");
-      outputWindowedValue(tag, output, null, null, PaneInfo.NO_FIRING);
+    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      return this;
     }
 
     @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      checkNotNull(tag, "TupleTag passed to outputWithTimestamp cannot be null");
-      outputWindowedValue(tag, output, timestamp, null, PaneInfo.NO_FIRING);
+    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Cannot access FinishBundleContext outside of @FinishBundle method.");
+    }
+
+    @Override
+    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Cannot access ProcessContext outside of @ProcessElement method.");
+    }
+
+    @Override
+    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Cannot access OnTimerContext outside of @OnTimer methods.");
+    }
+
+    @Override
+    public RestrictionTracker<?> restrictionTracker() {
+      throw new UnsupportedOperationException(
+          "Cannot access RestrictionTracker outside of @ProcessElement method.");
+    }
+
+    @Override
+    public State state(String stateId) {
+      throw new UnsupportedOperationException(
+          "Cannot access state outside of @ProcessElement and @OnTimer methods.");
+    }
+
+    @Override
+    public Timer timer(String timerId) {
+      throw new UnsupportedOperationException(
+          "Cannot access timers outside of @ProcessElement and @OnTimer methods.");
+    }
+  }
+
+  /**
+   * B
+   * A concrete implementation of {@link DoFn.FinishBundleContext}.
+   */
+  private class DoFnFinishBundleContext<InputT, OutputT>
+      extends DoFn<InputT, OutputT>.FinishBundleContext
+      implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+    private final DoFnContext<InputT, OutputT> context;
+
+    private DoFnFinishBundleContext(
+        DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+      fn.super();
+      this.context = context;
+    }
+
+    @Override
+    public PipelineOptions getPipelineOptions() {
+      return context.getPipelineOptions();
     }
 
     @Override
@@ -407,14 +484,20 @@ public BoundedWindow window() {
     }
 
     @Override
-    public Context context(DoFn<InputT, OutputT> doFn) {
+    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException(
+          "Cannot access StartBundleContext outside of @StartBundle method.");
+    }
+
+    @Override
+    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
       return this;
     }
 
     @Override
     public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
-          "Cannot access ProcessContext outside of @Processelement method.");
+          "Cannot access ProcessContext outside of @ProcessElement method.");
     }
 
     @Override
@@ -440,6 +523,17 @@ public Timer timer(String timerId) {
       throw new UnsupportedOperationException(
           "Cannot access timers outside of @ProcessElement and @OnTimer methods.");
     }
+
+    @Override
+    public void output(OutputT output, Instant timestamp, BoundedWindow window) {
+      context.outputWindowedValue(WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+    }
+
+    @Override
+    public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+      context.outputWindowedValue(
+          tag, WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+    }
   }
 
   /**
@@ -590,8 +684,13 @@ public BoundedWindow window() {
     }
 
     @Override
-    public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
-      return this;
+    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
+    }
+
+    @Override
+    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
     }
 
     @Override
@@ -698,15 +797,21 @@ public BoundedWindow window() {
     }
 
     @Override
-    public TimeDomain timeDomain() {
-      return timeDomain;
+    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
     }
 
     @Override
-    public Context context(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException("Context parameters are not supported.");
+    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+      throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
     }
 
+    @Override
+    public TimeDomain timeDomain() {
+      return timeDomain;
+    }
+
+
     @Override
     public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException("ProcessContext parameters are not supported.");
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
index ed065a67945a..7cf5e88a4c99 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
@@ -440,13 +440,13 @@ public void tearDown() throws Exception {
     }
 
     @StartBundle
-    public void startBundle(Context c) throws Exception {
-      invoker.invokeStartBundle(wrapContext(c));
+    public void startBundle(StartBundleContext c) throws Exception {
+      invoker.invokeStartBundle(wrapContextAsStartBundle(c));
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
-      invoker.invokeFinishBundle(wrapContext(c));
+    public void finishBundle(FinishBundleContext c) throws Exception {
+      invoker.invokeFinishBundle(wrapContextAsFinishBundle(c));
     }
 
     @ProcessElement
@@ -522,31 +522,39 @@ public void processElement(final ProcessContext c) {
               stateNamespace, timerInternals.currentProcessingTime(), TimeDomain.PROCESSING_TIME));
     }
 
-    private DoFn<InputT, OutputT>.Context wrapContext(final Context baseContext) {
-      return fn.new Context() {
+    private DoFn<InputT, OutputT>.StartBundleContext wrapContextAsStartBundle(
+        final StartBundleContext baseContext) {
+      return fn.new StartBundleContext() {
         @Override
         public PipelineOptions getPipelineOptions() {
           return baseContext.getPipelineOptions();
         }
 
-        @Override
-        public void output(OutputT output) {
-          throwUnsupportedOutput();
+        private void throwUnsupportedOutput() {
+          throw new UnsupportedOperationException(
+              String.format(
+                  "Splittable DoFn can only output from @%s",
+                  ProcessElement.class.getSimpleName()));
         }
+      };
+    }
 
+    private DoFn<InputT, OutputT>.FinishBundleContext wrapContextAsFinishBundle(
+        final FinishBundleContext baseContext) {
+      return fn.new FinishBundleContext() {
         @Override
-        public void outputWithTimestamp(OutputT output, Instant timestamp) {
+        public void output(OutputT output, Instant timestamp, BoundedWindow window) {
           throwUnsupportedOutput();
         }
 
         @Override
-        public <T> void output(TupleTag<T> tag, T output) {
+        public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
           throwUnsupportedOutput();
         }
 
         @Override
-        public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-          throwUnsupportedOutput();
+        public PipelineOptions getPipelineOptions() {
+          return baseContext.getPipelineOptions();
         }
 
         private void throwUnsupportedOutput() {
@@ -557,6 +565,7 @@ private void throwUnsupportedOutput() {
         }
       };
     }
+
   }
 
   /** Splits the restriction using the given {@link DoFn.SplitRestriction} method. */
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index 9b63bab94c1e..b8fc64ea7a55 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -348,12 +348,12 @@ static class ThrowingDoFn extends DoFn<String, String> {
     private static final TimerSpec timer = TimerSpecs.timer(TimeDomain.EVENT_TIME);
 
     @StartBundle
-    public void startBundle(Context c) throws Exception {
+    public void startBundle() throws Exception {
       throw exceptionToThrow;
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
+    public void finishBundle() throws Exception {
       throw exceptionToThrow;
     }
 
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
index a67db6d82ef9..0bb25fcfa747 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
@@ -578,13 +578,13 @@ public void tearDown() {
     }
 
     @StartBundle
-    public void startBundle(Context c) {
+    public void startBundle() {
       assertEquals(State.OUTSIDE_BUNDLE, state);
       state = State.INSIDE_BUNDLE;
     }
 
     @FinishBundle
-    public void finishBundle(Context c) {
+    public void finishBundle() {
       assertEquals(State.INSIDE_BUNDLE, state);
       state = State.OUTSIDE_BUNDLE;
     }
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index b0ae757b51aa..0367bd8a95a1 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170503</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170504</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index 64fe49504b2b..b85b315da869 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -1021,7 +1021,7 @@ static class ToIsmRecordForGlobalWindowDoFn<T>
 
       long indexInBundle;
       @StartBundle
-      public void startBundle(Context c) throws Exception {
+      public void startBundle() throws Exception {
         indexInBundle = 0;
       }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
index 2a057e4fbf09..5b219027e64c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
@@ -22,7 +22,6 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
-
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
@@ -246,6 +245,7 @@ private class WriteBundles extends DoFn<T, FileResult> {
     // Writer that will write the records in this bundle. Lazily
     // initialized in processElement.
     private FileBasedWriter<T> writer = null;
+    private BoundedWindow window = null;
 
     WriteBundles() {
     }
@@ -263,6 +263,7 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
         } else {
           writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, UNKNOWN_NUMSHARDS);
         }
+        this.window = window;
         LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
       }
       try {
@@ -285,10 +286,10 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
+    public void finishBundle(FinishBundleContext c) throws Exception {
       if (writer != null) {
         FileResult result = writer.close();
-        c.output(result);
+        c.output(result, window.maxTimestamp(), window);
         // Reset state in case of reuse.
         writer = null;
       }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
index d423bfadf03f..37081fde1a79 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
@@ -34,7 +34,6 @@
 import org.apache.beam.sdk.options.ProxyInvocationHandler.Serializer;
 import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.Context;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
 import org.joda.time.DateTimeUtils;
 import org.joda.time.DateTimeZone;
@@ -56,7 +55,7 @@
  * from command-line arguments with {@link PipelineOptionsFactory#fromArgs(String[])}.
  * They can be converted to another type by invoking {@link PipelineOptions#as(Class)} and
  * can be accessed from within a {@link DoFn} by invoking
- * {@link Context#getPipelineOptions()}.
+ * {@code getPipelineOptions()} on the input {@link DoFn.ProcessContext Context} object.
  *
  * <p>For example:
  * <pre>{@code
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 0be8517da9dc..9d0e97ab16aa 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -1844,9 +1844,8 @@ public void populateDisplayData(DisplayData.Builder builder) {
           new DoFn<KV<K, InputT>, KV<K, InputT>>() {
             transient int counter;
             @StartBundle
-            public void startBundle(Context c) {
-              counter = ThreadLocalRandom.current().nextInt(
-                  Integer.MAX_VALUE);
+            public void startBundle() {
+              counter = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE);
             }
 
             @ProcessElement
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index eab08f6b8793..f3d178e16f84 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -82,15 +82,62 @@
  * @param <OutputT> the type of the (main) output elements
  */
 public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayData {
+  /**
+   * Information accessible while within the {@link StartBundle} method.
+   */
+  public abstract class StartBundleContext {
+    /**
+     * Returns the {@code PipelineOptions} specified with the {@link
+     * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+     * PipelineOptions} will be the default running via {@link DoFnTester}.
+     */
+    public abstract PipelineOptions getPipelineOptions();
+  }
+
+  /**
+   * Information accessible while within the {@link FinishBundle} method.
+   */
+  public abstract class FinishBundleContext {
+    /**
+     * Returns the {@code PipelineOptions} specified with the {@link
+     * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+     * PipelineOptions} will be the default running via {@link DoFnTester}.
+     */
+    public abstract PipelineOptions getPipelineOptions();
+
+    /**
+     * Adds the given element to the main output {@code PCollection} at the given
+     * timestamp in the given window.
+     *
+     * <p>Once passed to {@code output} the element should not be modified in
+     * any way.
+     *
+     * <p><i>Note:</i> A splittable {@link DoFn} is not allowed to output from the
+     * {@link FinishBundle} method.
+     */
+    public abstract void output(OutputT output, Instant timestamp, BoundedWindow window);
 
-  /** Information accessible to all methods in this {@code DoFn}. */
-  public abstract class Context {
+    /**
+     * Adds the given element to the output {@code PCollection} with the given tag at the given
+     * timestamp in the given window.
+     *
+     * <p>Once passed to {@code output} the element should not be modified in any way.
+     *
+     * <p><i>Note:</i> A splittable {@link DoFn} is not allowed to output from the {@link
+     * FinishBundle} method.
+     */
+    public abstract <T> void output(
+        TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window);
+  }
 
+  /**
+   * Information accessible to all methods in this {@link DoFn} where the context is in some window.
+   */
+  public abstract class WindowedContext {
     /**
-     * Returns the {@code PipelineOptions} specified with the
-     * {@link org.apache.beam.sdk.runners.PipelineRunner}
-     * invoking this {@code DoFn}.  The {@code PipelineOptions} will
-     * be the default running via {@link DoFnTester}.
+     * Returns the {@code PipelineOptions} specified with the {@link
+     * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+     * PipelineOptions} will be the default running via {@link DoFnTester}.
      */
     public abstract PipelineOptions getPipelineOptions();
 
@@ -205,7 +252,7 @@ public abstract <T> void outputWithTimestamp(
   /**
    * Information accessible when running a {@link DoFn.ProcessElement} method.
    */
-  public abstract class ProcessContext extends Context {
+  public abstract class ProcessContext extends WindowedContext {
 
     /**
      * Returns the input element to be processed.
@@ -215,7 +262,6 @@ public abstract class ProcessContext extends Context {
      */
     public abstract InputT element();
 
-
     /**
      * Returns the value of the side input.
      *
@@ -257,7 +303,7 @@ public abstract class ProcessContext extends Context {
   /**
    * Information accessible when running a {@link DoFn.OnTimer} method.
    */
-  public abstract class OnTimerContext extends Context {
+  public abstract class OnTimerContext extends WindowedContext {
 
     /**
      * Returns the timestamp of the current timer.
@@ -277,7 +323,7 @@ public abstract class OnTimerContext extends Context {
 
   /**
    * Returns the allowed timestamp skew duration, which is the maximum duration that timestamps can
-   * be shifted backward in {@link DoFn.Context#outputWithTimestamp}.
+   * be shifted backward in {@link WindowedContext#outputWithTimestamp}.
    *
    * <p>The default value is {@code Duration.ZERO}, in which case timestamps can only be shifted
    * forward to future. For infinite skew, return {@code Duration.millis(Long.MAX_VALUE)}.
@@ -449,14 +495,9 @@ public interface OutputReceiver<T> {
    * Annotation for the method to use to prepare an instance for processing a batch of elements.
    * The method annotated with this must satisfy the following constraints:
    * <ul>
-   *   <li>It must have exactly one argument.
-   *   <li>Its first (and only) argument must be a {@link DoFn.Context}.
+   *   <li>It must have exactly zero or one arguments.
+   *   <li>If it has any arguments, its only argument must be a {@link DoFn.StartBundleContext}.
    * </ul>
-   *
-   * <p>A simple method declaration would look like:
-   * <code>
-   *   public void setup(DoFn.Context c) { .. }
-   * </code>
    */
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
@@ -524,8 +565,8 @@ public interface OutputReceiver<T> {
    * Annotation for the method to use to finish processing a batch of elements.
    * The method annotated with this must satisfy the following constraints:
    * <ul>
-   *   <li>It must have at least one argument.
-   *   <li>Its first (and only) argument must be a {@link DoFn.Context}.
+   *   <li>It must have exactly zero or one arguments.
+   *   <li>If it has any arguments, its only argument must be a {@link DoFn.FinishBundleContext}.
    * </ul>
    */
   @Documented
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index 70fb0ae9cb0b..ca7427c9bc64 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -218,9 +218,8 @@ public void startBundle() throws Exception {
     if (state == State.UNINITIALIZED) {
       initializeState();
     }
-    TestContext context = new TestContext();
     try {
-      fnInvoker.invokeStartBundle(context);
+      fnInvoker.invokeStartBundle(new TestStartBundleContext());
     } catch (UserCodeException e) {
       unwrapUserCodeException(e);
     }
@@ -291,9 +290,17 @@ public BoundedWindow window() {
             }
 
             @Override
-            public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
+            public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(
+                DoFn<InputT, OutputT> doFn) {
               throw new UnsupportedOperationException(
-                  "Not expected to access DoFn.Context from @ProcessElement");
+                  "Not expected to access DoFn.StartBundleContext from @ProcessElement");
+            }
+
+            @Override
+            public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+                DoFn<InputT, OutputT> doFn) {
+              throw new UnsupportedOperationException(
+                  "Not expected to access DoFn.FinishBundleContext from @ProcessElement");
             }
 
             @Override
@@ -303,8 +310,7 @@ public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT>
 
             @Override
             public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
-              throw new UnsupportedOperationException(
-                  "DoFnTester doesn't support timers yet.");
+              throw new UnsupportedOperationException("DoFnTester doesn't support timers yet.");
             }
 
             @Override
@@ -344,7 +350,7 @@ public void finishBundle() throws Exception {
         "Must be inside bundle to call finishBundle, but was: %s",
         state);
     try {
-      fnInvoker.invokeFinishBundle(new TestContext());
+      fnInvoker.invokeFinishBundle(new TestFinishBundleContext());
     } catch (UserCodeException e) {
       unwrapUserCodeException(e);
     }
@@ -522,8 +528,9 @@ public TupleTag<OutputT> getMainOutputTag() {
     return mainOutputTag;
   }
 
-  private class TestContext extends DoFn<InputT, OutputT>.Context {
-    TestContext() {
+  private class TestStartBundleContext extends DoFn<InputT, OutputT>.StartBundleContext {
+
+    private TestStartBundleContext() {
       fn.super();
     }
 
@@ -531,30 +538,33 @@ private class TestContext extends DoFn<InputT, OutputT>.Context {
     public PipelineOptions getPipelineOptions() {
       return options;
     }
+  }
 
-    @Override
-    public void output(OutputT output) {
-      throwUnsupportedOutputFromBundleMethods();
+  private class TestFinishBundleContext extends DoFn<InputT, OutputT>.FinishBundleContext {
+
+    private TestFinishBundleContext() {
+      fn.super();
     }
 
-    @Override
-    public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      throwUnsupportedOutputFromBundleMethods();
+    private void throwUnsupportedOutputFromBundleMethods() {
+      throw new UnsupportedOperationException(
+          "DoFnTester doesn't support output from bundle methods");
     }
 
     @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      throwUnsupportedOutputFromBundleMethods();
+    public PipelineOptions getPipelineOptions() {
+      return options;
     }
 
     @Override
-    public <T> void output(TupleTag<T> tag, T output) {
+    public void output(
+        OutputT output, Instant timestamp, BoundedWindow window) {
       throwUnsupportedOutputFromBundleMethods();
     }
 
-    private void throwUnsupportedOutputFromBundleMethods() {
-      throw new UnsupportedOperationException(
-          "DoFnTester doesn't support output from bundle methods");
+    @Override
+    public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+      throwUnsupportedOutputFromBundleMethods();
     }
   }
 
@@ -564,12 +574,10 @@ public DoFn<InputT, OutputT>.ProcessContext createProcessContext(
   }
 
   private class TestProcessContext extends DoFn<InputT, OutputT>.ProcessContext {
-    private final TestContext context;
     private final ValueInSingleWindow<InputT> element;
 
     private TestProcessContext(ValueInSingleWindow<InputT> element) {
       fn.super();
-      this.context = new TestContext();
       this.element = element;
     }
 
@@ -611,7 +619,7 @@ public void updateWatermark(Instant watermark) {
 
     @Override
     public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
+      return options;
     }
 
     @Override
@@ -634,6 +642,12 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
       getMutableOutput(tag)
           .add(ValueInSingleWindow.of(output, timestamp, element.getWindow(), element.getPane()));
     }
+
+    private void throwUnsupportedOutputFromBundleMethods() {
+      throw new UnsupportedOperationException(
+          "DoFnTester doesn't support output from bundle methods");
+    }
+
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
index 564baf4e8b3b..e1321151ada0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
@@ -195,7 +195,7 @@ public void onTimerCallback(
     }
 
     private void flushBatch(
-        Context c,
+        WindowedContext c,
         ValueState<K> key,
         BagState<InputT> batch,
         CombiningState<Long, long[], Long> numElementsInBatch) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index 73d78c965e92..7dd2cdd53ce8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -33,6 +33,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
 import org.apache.beam.sdk.transforms.display.DisplayData.ItemSpec;
@@ -191,8 +192,8 @@
  * necessarily need to be explicitly specified, even if the {@link DoFn}
  * generates them. Within the {@link DoFn}, an element is added to the
  * main output {@link PCollection} as normal, using
- * {@link DoFn.Context#output(Object)}, while an element is added to any additional output
- * {@link PCollection} using {@link DoFn.Context#output(TupleTag, Object)}. For example:
+ * {@link WindowedContext#output(Object)}, while an element is added to any additional output
+ * {@link PCollection} using {@link WindowedContext#output(TupleTag, Object)}. For example:
  *
  * <pre>{@code
  * PCollection<String> words = ...;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
index 6bef4df68648..2b6d5be1c595 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
@@ -67,10 +67,11 @@
 import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.OnTimerMethod;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.Cases;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.ContextParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.FinishBundleContextParameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.OnTimerContextParameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.ProcessContextParameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StartBundleContextParameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
@@ -84,6 +85,8 @@
 public class ByteBuddyDoFnInvokerFactory implements DoFnInvokerFactory {
 
   public static final String CONTEXT_PARAMETER_METHOD = "context";
+  public static final String START_BUNDLE_CONTEXT_PARAMETER_METHOD = "startBundleContext";
+  public static final String FINISH_BUNDLE_CONTEXT_PARAMETER_METHOD = "finishBundleContext";
   public static final String PROCESS_CONTEXT_PARAMETER_METHOD = "processContext";
   public static final String ON_TIMER_CONTEXT_PARAMETER_METHOD = "onTimerContext";
   public static final String WINDOW_PARAMETER_METHOD = "window";
@@ -557,11 +560,21 @@ static StackManipulation getExtraContextParameter(
         new Cases<StackManipulation>() {
 
           @Override
-          public StackManipulation dispatch(ContextParameter p) {
+          public StackManipulation dispatch(StartBundleContextParameter p) {
             return new StackManipulation.Compound(
                 pushDelegate,
                 MethodInvocation.invoke(
-                    getExtraContextFactoryMethodDescription(CONTEXT_PARAMETER_METHOD, DoFn.class)));
+                    getExtraContextFactoryMethodDescription(
+                        START_BUNDLE_CONTEXT_PARAMETER_METHOD, DoFn.class)));
+          }
+
+          @Override
+          public StackManipulation dispatch(FinishBundleContextParameter p) {
+            return new StackManipulation.Compound(
+                pushDelegate,
+                MethodInvocation.invoke(
+                    getExtraContextFactoryMethodDescription(
+                        FINISH_BUNDLE_CONTEXT_PARAMETER_METHOD, DoFn.class)));
           }
 
           @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
index 0fbcc84cfd9a..3c44afe7b98f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
@@ -41,10 +41,10 @@ public interface DoFnInvoker<InputT, OutputT> {
   void invokeSetup();
 
   /** Invoke the {@link DoFn.StartBundle} method on the bound {@link DoFn}. */
-  void invokeStartBundle(DoFn<InputT, OutputT>.Context c);
+  void invokeStartBundle(DoFn<InputT, OutputT>.StartBundleContext c);
 
   /** Invoke the {@link DoFn.FinishBundle} method on the bound {@link DoFn}. */
-  void invokeFinishBundle(DoFn<InputT, OutputT>.Context c);
+  void invokeFinishBundle(DoFn<InputT, OutputT>.FinishBundleContext c);
 
   /** Invoke the {@link DoFn.Teardown} method on the bound {@link DoFn}. */
   void invokeTeardown();
@@ -100,8 +100,11 @@ interface ArgumentProvider<InputT, OutputT> {
      */
     BoundedWindow window();
 
-    /** Provide a {@link DoFn.Context} to use with the given {@link DoFn}. */
-    DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn);
+    /** Provide a {@link DoFn.StartBundleContext} to use with the given {@link DoFn}. */
+    DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn);
+
+    /** Provide a {@link DoFn.FinishBundleContext} to use with the given {@link DoFn}. */
+    DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn);
 
     /** Provide a {@link DoFn.ProcessContext} to use with the given {@link DoFn}. */
     DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn);
@@ -135,7 +138,13 @@ public BoundedWindow window() {
     }
 
     @Override
-    public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+      return null;
+    }
+
+    @Override
+    public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+        DoFn<InputT, OutputT> doFn) {
       return null;
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index 1be741f2db39..3219f9680a55 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -177,8 +177,10 @@ private Parameter() {}
     public <ResultT> ResultT match(Cases<ResultT> cases) {
       // This could be done with reflection, but since the number of cases is small and known,
       // they are simply inlined.
-      if (this instanceof ContextParameter) {
-        return cases.dispatch((ContextParameter) this);
+      if (this instanceof StartBundleContextParameter) {
+        return cases.dispatch((StartBundleContextParameter) this);
+      } else if (this instanceof FinishBundleContextParameter) {
+        return cases.dispatch((FinishBundleContextParameter) this);
       } else if (this instanceof ProcessContextParameter) {
         return cases.dispatch((ProcessContextParameter) this);
       } else if (this instanceof OnTimerContextParameter) {
@@ -202,7 +204,8 @@ public <ResultT> ResultT match(Cases<ResultT> cases) {
      * An interface for destructuring a {@link Parameter}.
      */
     public interface Cases<ResultT> {
-      ResultT dispatch(ContextParameter p);
+      ResultT dispatch(StartBundleContextParameter p);
+      ResultT dispatch(FinishBundleContextParameter p);
       ResultT dispatch(ProcessContextParameter p);
       ResultT dispatch(OnTimerContextParameter p);
       ResultT dispatch(WindowParameter p);
@@ -218,7 +221,12 @@ abstract class WithDefault<ResultT> implements Cases<ResultT> {
         protected abstract ResultT dispatchDefault(Parameter p);
 
         @Override
-        public ResultT dispatch(ContextParameter p) {
+        public ResultT dispatch(StartBundleContextParameter p) {
+          return dispatchDefault(p);
+        }
+
+        @Override
+        public ResultT dispatch(FinishBundleContextParameter p) {
           return dispatchDefault(p);
         }
 
@@ -255,18 +263,15 @@ public ResultT dispatch(TimerParameter p) {
     }
 
     // These parameter descriptors are constant
-    private static final ContextParameter CONTEXT_PARAMETER =
-        new AutoValue_DoFnSignature_Parameter_ContextParameter();
+    private static final StartBundleContextParameter START_BUNDLE_CONTEXT_PARAMETER =
+        new AutoValue_DoFnSignature_Parameter_StartBundleContextParameter();
+    private static final FinishBundleContextParameter FINISH_BUNDLE_CONTEXT_PARAMETER =
+        new AutoValue_DoFnSignature_Parameter_FinishBundleContextParameter();
     private static final ProcessContextParameter PROCESS_CONTEXT_PARAMETER =
           new AutoValue_DoFnSignature_Parameter_ProcessContextParameter();
     private static final OnTimerContextParameter ON_TIMER_CONTEXT_PARAMETER =
         new AutoValue_DoFnSignature_Parameter_OnTimerContextParameter();
 
-    /** Returns a {@link ContextParameter}. */
-    public static ContextParameter context() {
-      return CONTEXT_PARAMETER;
-    }
-
     /** Returns a {@link ProcessContextParameter}. */
     public static ProcessContextParameter processContext() {
       return PROCESS_CONTEXT_PARAMETER;
@@ -301,13 +306,22 @@ public static TimerParameter timerParameter(TimerDeclaration decl) {
     }
 
     /**
-     * Descriptor for a {@link Parameter} of type {@link DoFn.Context}.
+     * Descriptor for a {@link Parameter} of type {@link DoFn.StartBundleContext}.
+     *
+     * <p>All such descriptors are equal.
+     */
+    @AutoValue
+    public abstract static class StartBundleContextParameter extends Parameter {
+      StartBundleContextParameter() {}
+    }
+    /**
+     * Descriptor for a {@link Parameter} of type {@link DoFn.FinishBundleContext}.
      *
      * <p>All such descriptors are equal.
      */
     @AutoValue
-    public abstract static class ContextParameter extends Parameter {
-      ContextParameter() {}
+    public abstract static class FinishBundleContextParameter extends Parameter {
+      FinishBundleContextParameter() {}
     }
 
     /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
index 666c7f69b3b7..bac3befc36df 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -346,14 +346,14 @@ private static DoFnSignature parseSignature(Class<? extends DoFn<?, ?>> fnClass)
     if (startBundleMethod != null) {
       ErrorReporter startBundleErrors = errors.forMethod(DoFn.StartBundle.class, startBundleMethod);
       signatureBuilder.setStartBundle(
-          analyzeBundleMethod(startBundleErrors, fnT, startBundleMethod, inputT, outputT));
+          analyzeStartBundleMethod(startBundleErrors, fnT, startBundleMethod, inputT, outputT));
     }
 
     if (finishBundleMethod != null) {
       ErrorReporter finishBundleErrors =
           errors.forMethod(DoFn.FinishBundle.class, finishBundleMethod);
       signatureBuilder.setFinishBundle(
-          analyzeBundleMethod(finishBundleErrors, fnT, finishBundleMethod, inputT, outputT));
+          analyzeFinishBundleMethod(finishBundleErrors, fnT, finishBundleMethod, inputT, outputT));
     }
 
     if (setupMethod != null) {
@@ -607,12 +607,25 @@ TypeDescriptor<DoFn<InputT, OutputT>.ProcessContext> doFnProcessContextTypeOf(
   }
 
   /**
-   * Generates a {@link TypeDescriptor} for {@code DoFn<InputT, OutputT>.Context} given {@code
-   * InputT} and {@code OutputT}.
+   * Generates a {@link TypeDescriptor} for {@code DoFn<InputT, OutputT>.StartBundleContext} given
+   * {@code InputT} and {@code OutputT}.
+   */
+  private static <InputT, OutputT>
+      TypeDescriptor<DoFn<InputT, OutputT>.StartBundleContext> doFnStartBundleContextTypeOf(
+          TypeDescriptor<InputT> inputT, TypeDescriptor<OutputT> outputT) {
+    return new TypeDescriptor<DoFn<InputT, OutputT>.StartBundleContext>() {}.where(
+            new TypeParameter<InputT>() {}, inputT)
+        .where(new TypeParameter<OutputT>() {}, outputT);
+  }
+
+  /**
+   * Generates a {@link TypeDescriptor} for {@code DoFn<InputT, OutputT>.FinishBundleContext} given
+   * {@code InputT} and {@code OutputT}.
    */
-  private static <InputT, OutputT> TypeDescriptor<DoFn<InputT, OutputT>.Context> doFnContextTypeOf(
-      TypeDescriptor<InputT> inputT, TypeDescriptor<OutputT> outputT) {
-    return new TypeDescriptor<DoFn<InputT, OutputT>.Context>() {}.where(
+  private static <InputT, OutputT>
+      TypeDescriptor<DoFn<InputT, OutputT>.FinishBundleContext> doFnFinishBundleContextTypeOf(
+          TypeDescriptor<InputT> inputT, TypeDescriptor<OutputT> outputT) {
+    return new TypeDescriptor<DoFn<InputT, OutputT>.FinishBundleContext>() {}.where(
             new TypeParameter<InputT>() {}, inputT)
         .where(new TypeParameter<OutputT>() {}, outputT);
   }
@@ -752,7 +765,6 @@ private static Parameter analyzeExtraParameter(
       TypeDescriptor<?> outputT) {
 
     TypeDescriptor<?> expectedProcessContextT = doFnProcessContextTypeOf(inputT, outputT);
-    TypeDescriptor<?> expectedContextT = doFnContextTypeOf(inputT, outputT);
     TypeDescriptor<?> expectedOnTimerContextT = doFnOnTimerContextTypeOf(inputT, outputT);
 
     TypeDescriptor<?> paramT = param.getType();
@@ -765,11 +777,6 @@ private static Parameter analyzeExtraParameter(
         "ProcessContext argument must have type %s",
         formatType(expectedProcessContextT));
       return Parameter.processContext();
-    } else if (rawType.equals(DoFn.Context.class)) {
-      paramErrors.checkArgument(paramT.equals(expectedContextT),
-          "Context argument must have type %s",
-          formatType(expectedContextT));
-      return Parameter.context();
     } else if (rawType.equals(DoFn.OnTimerContext.class)) {
       paramErrors.checkArgument(
           paramT.equals(expectedOnTimerContextT),
@@ -921,17 +928,36 @@ private static TypeDescriptor<? extends BoundedWindow> getWindowType(
   }
 
   @VisibleForTesting
-  static DoFnSignature.BundleMethod analyzeBundleMethod(
+  static DoFnSignature.BundleMethod analyzeStartBundleMethod(
+      ErrorReporter errors,
+      TypeDescriptor<? extends DoFn<?, ?>> fnT,
+      Method m,
+      TypeDescriptor<?> inputT,
+      TypeDescriptor<?> outputT) {
+    errors.checkArgument(void.class.equals(m.getReturnType()), "Must return void");
+    TypeDescriptor<?> expectedContextT = doFnStartBundleContextTypeOf(inputT, outputT);
+    Type[] params = m.getGenericParameterTypes();
+    errors.checkArgument(
+        params.length == 0
+            || (params.length == 1 && fnT.resolveType(params[0]).equals(expectedContextT)),
+        "Must take a single argument of type %s",
+        formatType(expectedContextT));
+    return DoFnSignature.BundleMethod.create(m);
+  }
+
+  @VisibleForTesting
+  static DoFnSignature.BundleMethod analyzeFinishBundleMethod(
       ErrorReporter errors,
       TypeDescriptor<? extends DoFn<?, ?>> fnT,
       Method m,
       TypeDescriptor<?> inputT,
       TypeDescriptor<?> outputT) {
     errors.checkArgument(void.class.equals(m.getReturnType()), "Must return void");
-    TypeDescriptor<?> expectedContextT = doFnContextTypeOf(inputT, outputT);
+    TypeDescriptor<?> expectedContextT = doFnFinishBundleContextTypeOf(inputT, outputT);
     Type[] params = m.getGenericParameterTypes();
     errors.checkArgument(
-        params.length == 1 && fnT.resolveType(params[0]).equals(expectedContextT),
+        params.length == 0
+            || (params.length == 1 && fnT.resolveType(params[0]).equals(expectedContextT)),
         "Must take a single argument of type %s",
         formatType(expectedContextT));
     return DoFnSignature.BundleMethod.create(m);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
index b43c7a3a7836..faf3ca93612c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
@@ -30,6 +30,7 @@
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.util.VarInt;
 
@@ -70,7 +71,7 @@ public final class PaneInfo {
    * definitions:
    * <ol>
    * <li>We'll call a pipeline 'simple' if it does not use
-   * {@link DoFn.Context#outputWithTimestamp} in
+   * {@link WindowedContext#outputWithTimestamp} in
    * any {@link DoFn}, and it uses the same
    * {@link org.apache.beam.sdk.transforms.windowing.Window#withAllowedLateness}
    * argument value on all windows (or uses the default of {@link org.joda.time.Duration#ZERO}).
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
index 8077c27cd7b2..084c44508a75 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
@@ -210,7 +210,7 @@ private PipelineResult runPipelineWithMetrics() {
           Distribution bundleDist = Metrics.distribution(MetricsTest.class, "bundle");
 
           @StartBundle
-          public void startBundle(Context c) {
+          public void startBundle() {
             bundleDist.update(10L);
           }
 
@@ -226,7 +226,7 @@ public void processElement(ProcessContext c) {
           }
 
           @DoFn.FinishBundle
-          public void finishBundle(Context c) {
+          public void finishBundle() {
             bundleDist.update(40L);
           }
         }))
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
index d609d0e5f257..a980f87b5c74 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
@@ -195,12 +195,12 @@ public void process(ProcessContext context) {
     }
 
     @StartBundle
-    public void startBundle(Context context) {
+    public void startBundle() {
       ++numStartBundleCalls;
     }
 
     @FinishBundle
-    public void finishBundle(Context context) {
+    public void finishBundle() {
       ++numFinishBundleCalls;
     }
   }
@@ -397,7 +397,7 @@ public void setup() {
     }
 
     @StartBundle
-    public void startBundle(Context c) {
+    public void startBundle() {
       checkState(state == LifecycleState.SET_UP, "Wrong state: %s", state);
       state = LifecycleState.INSIDE_BUNDLE;
       startBundleCalls.inc();
@@ -412,7 +412,7 @@ public void processElement(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) {
+    public void finishBundle() {
       checkState(state == LifecycleState.INSIDE_BUNDLE, "Wrong state: %s", state);
       state = LifecycleState.SET_UP;
       finishBundleCalls.inc();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
index fda8947f067d..849b874852d5 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
@@ -87,7 +87,7 @@ public void setup() {
     }
 
     @StartBundle
-    public void startBundle(Context c) {
+    public void startBundle() {
       assertThat("setup should have been called", setupCalled, is(true));
       assertThat(
           "Even number of startBundle and finishBundle calls in startBundle",
@@ -108,7 +108,7 @@ public void processElement(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) {
+    public void finishBundle() {
       assertThat("startBundle should have been called", startBundleCalls, greaterThan(0));
       assertThat(
           "there should be one bundle that has been started but not finished",
@@ -167,7 +167,7 @@ public void before() {
     }
 
     @StartBundle
-    public void begin(Context c) {
+    public void begin() {
       assertThat("setup should have been called", setupCalled, is(true));
       assertThat("Even number of startBundle and finishBundle calls in startBundle",
           startBundleCalls,
@@ -186,7 +186,7 @@ public void process(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void end(Context c) {
+    public void end() {
       assertThat("startBundle should have been called", startBundleCalls, greaterThan(0));
       assertThat("there should be one bundle that has been started but not finished",
           startBundleCalls,
@@ -354,7 +354,7 @@ public void setup() throws Exception {
     }
 
     @StartBundle
-    public void startBundle(Context c) throws Exception {
+    public void startBundle() throws Exception {
       throwIfNecessary(MethodForException.START_BUNDLE);
     }
 
@@ -364,7 +364,7 @@ public void processElement(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
+    public void finishBundle() throws Exception {
       throwIfNecessary(MethodForException.FINISH_BUNDLE);
     }
 
@@ -401,7 +401,7 @@ public void before() throws Exception {
     }
 
     @StartBundle
-    public void preBundle(Context c) throws Exception {
+    public void preBundle() throws Exception {
       throwIfNecessary(MethodForException.START_BUNDLE);
     }
 
@@ -411,7 +411,7 @@ public void perElement(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void postBundle(Context c) throws Exception {
+    public void postBundle() throws Exception {
       throwIfNecessary(MethodForException.FINISH_BUNDLE);
     }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 1c919d4fb659..56051a69689d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -170,12 +170,11 @@ public void prepare() {
     }
 
     @StartBundle
-    public void startBundle(Context c) {
+    public void startBundle() {
       assertThat(state,
           anyOf(equalTo(State.UNSTARTED), equalTo(State.FINISHED)));
 
       state = State.STARTED;
-      outputToAll(c, "started");
     }
 
     @ProcessElement
@@ -187,18 +186,17 @@ public void processElement(ProcessContext c) {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) {
+    public void finishBundle(FinishBundleContext c) {
       assertThat(state,
                  anyOf(equalTo(State.STARTED), equalTo(State.PROCESSING)));
       state = State.FINISHED;
-      outputToAll(c, "finished");
-    }
-
-    private void outputToAll(Context c, String value) {
-      c.output(value);
+      c.output("finished", BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE);
       for (TupleTag<String> additionalOutputTupleTag : additionalOutputTupleTags) {
-        c.output(additionalOutputTupleTag,
-                     additionalOutputTupleTag.getId() + ": " + value);
+        c.output(
+            additionalOutputTupleTag,
+            additionalOutputTupleTag.getId() + ": " + "finished",
+            BoundedWindow.TIMESTAMP_MIN_VALUE,
+            GlobalWindow.INSTANCE);
       }
     }
 
@@ -220,7 +218,7 @@ private void outputToAllWithSideInputs(ProcessContext c, String value) {
 
   static class TestStartBatchErrorDoFn extends DoFn<Integer, String> {
     @StartBundle
-    public void startBundle(Context c) {
+    public void startBundle() {
       throw new RuntimeException("test error in initialize");
     }
 
@@ -244,7 +242,7 @@ public void processElement(ProcessContext c) {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) {
+    public void finishBundle(FinishBundleContext c) {
       throw new RuntimeException("test error in finalize");
     }
   }
@@ -1156,13 +1154,10 @@ public HasExpectedOutput inOrder() {
 
     @Override
     public Void apply(Iterable<String> outputs) {
-      List<String> starteds = new ArrayList<>();
       List<String> processeds = new ArrayList<>();
       List<String> finisheds = new ArrayList<>();
       for (String output : outputs) {
-        if (output.contains("started")) {
-          starteds.add(output);
-        } else if (output.contains("finished")) {
+        if (output.contains("finished")) {
           finisheds.add(output);
         } else {
           processeds.add(output);
@@ -1196,10 +1191,6 @@ public Void apply(Iterable<String> outputs) {
         assertThat(processeds, contains(expectedProcessedsArray));
       }
 
-      assertEquals(starteds.size(), finisheds.size());
-      for (String started : starteds) {
-        assertEquals(additionalOutputPrefix + "started", started);
-      }
       for (String finished : finisheds) {
         assertEquals(additionalOutputPrefix + "finished", finished);
       }
@@ -1483,7 +1474,6 @@ public Void apply(Iterable<Long> input) {
   private static class Checker implements SerializableFunction<Iterable<String>, Void> {
     @Override
     public Void apply(Iterable<String> input) {
-      boolean foundStart = false;
       boolean foundElement = false;
       boolean foundFinish = false;
       for (String str : input) {
@@ -1492,17 +1482,12 @@ public Void apply(Iterable<String> input) {
             throw new AssertionError("Received duplicate element");
           }
           foundElement = true;
-        } else if (str.equals("start:2:2")) {
-          foundStart = true;
         } else if (str.equals("finish:3:3")) {
           foundFinish = true;
         } else {
           throw new AssertionError("Got unexpected value: " + str);
         }
       }
-      if (!foundStart) {
-        throw new AssertionError("Missing \"start:2:2\"");
-      }
       if (!foundElement) {
         throw new AssertionError("Missing \"elem:1:1\"");
       }
@@ -1518,19 +1503,14 @@ public Void apply(Iterable<String> input) {
   @Category(ValidatesRunner.class)
   public void testWindowingInStartAndFinishBundle() {
 
+    final FixedWindows windowFn = FixedWindows.of(Duration.millis(1));
     PCollection<String> output =
         pipeline
             .apply(Create.timestamped(TimestampedValue.of("elem", new Instant(1))))
-            .apply(Window.<String>into(FixedWindows.of(Duration.millis(1))))
+            .apply(Window.<String>into(windowFn))
             .apply(
                 ParDo.of(
                     new DoFn<String, String>() {
-                      @StartBundle
-                      public void startBundle(Context c) {
-                        c.outputWithTimestamp("start", new Instant(2));
-                        System.out.println("Start: 2");
-                      }
-
                       @ProcessElement
                       public void processElement(ProcessContext c) {
                         c.output(c.element());
@@ -1539,8 +1519,9 @@ public void processElement(ProcessContext c) {
                       }
 
                       @FinishBundle
-                      public void finishBundle(Context c) {
-                        c.outputWithTimestamp("finish", new Instant(3));
+                      public void finishBundle(FinishBundleContext c) {
+                        Instant ts = new Instant(3);
+                        c.output("finish", ts, windowFn.assignWindow(ts));
                         System.out.println("Finish: 3");
                       }
                     }))
@@ -1551,30 +1532,6 @@ public void finishBundle(Context c) {
     pipeline.run();
   }
 
-  @Test
-  @Category(NeedsRunner.class)
-  public void testWindowingInStartBundleException() {
-
-    pipeline
-        .apply(Create.timestamped(TimestampedValue.of("elem", new Instant(1))))
-        .apply(Window.<String>into(FixedWindows.of(Duration.millis(1))))
-        .apply(
-            ParDo.of(
-                new DoFn<String, String>() {
-                  @StartBundle
-                  public void startBundle(Context c) {
-                    c.output("start");
-                  }
-
-                  @ProcessElement
-                  public void processElement(ProcessContext c) {
-                    c.output(c.element());
-                  }
-                }));
-
-    thrown.expectMessage("WindowFn attempted to access input timestamp when none was available");
-    pipeline.run();
-  }
   @Test
   public void testDoFnDisplayData() {
     DoFn<String, String> fn = new DoFn<String, String>() {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
index a0f1fd3573f4..02a44d2b907e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
@@ -405,13 +405,13 @@ public void setUp() {
     }
 
     @StartBundle
-    public void startBundle(Context c) {
+    public void startBundle() {
       assertEquals(State.OUTSIDE_BUNDLE, state);
       state = State.INSIDE_BUNDLE;
     }
 
     @FinishBundle
-    public void finishBundle(Context c) {
+    public void finishBundle() {
       assertEquals(State.INSIDE_BUNDLE, state);
       state = State.OUTSIDE_BUNDLE;
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index c16eea2ba5c9..fe96e87951a4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -71,6 +71,8 @@
 public class DoFnInvokersTest {
   @Rule public ExpectedException thrown = ExpectedException.none();
 
+  @Mock private DoFn<String, String>.StartBundleContext mockStartBundleContext;
+  @Mock private DoFn<String, String>.FinishBundleContext mockFinishBundleContext;
   @Mock private DoFn<String, String>.ProcessContext mockProcessContext;
   @Mock private IntervalWindow mockWindow;
   @Mock private DoFnInvoker.ArgumentProvider<String, String> mockArgumentProvider;
@@ -79,6 +81,10 @@ public class DoFnInvokersTest {
   public void setUp() {
     MockitoAnnotations.initMocks(this);
     when(mockArgumentProvider.window()).thenReturn(mockWindow);
+    when(mockArgumentProvider.startBundleContext(Matchers.<DoFn>any()))
+        .thenReturn(mockStartBundleContext);
+    when(mockArgumentProvider.finishBundleContext(Matchers.<DoFn>any()))
+        .thenReturn(mockFinishBundleContext);
     when(mockArgumentProvider.processContext(Matchers.<DoFn>any())).thenReturn(mockProcessContext);
   }
 
@@ -233,10 +239,10 @@ class MockFn extends DoFn<String, String> {
       public void processElement(ProcessContext c) {}
 
       @StartBundle
-      public void startBundle(Context c) {}
+      public void startBundle(StartBundleContext c) {}
 
       @FinishBundle
-      public void finishBundle(Context c) {}
+      public void finishBundle(FinishBundleContext c) {}
 
       @Setup
       public void before() {}
@@ -247,12 +253,12 @@ public void after() {}
     MockFn fn = mock(MockFn.class);
     DoFnInvoker<String, String> invoker = DoFnInvokers.invokerFor(fn);
     invoker.invokeSetup();
-    invoker.invokeStartBundle(mockProcessContext);
-    invoker.invokeFinishBundle(mockProcessContext);
+    invoker.invokeStartBundle(mockStartBundleContext);
+    invoker.invokeFinishBundle(mockFinishBundleContext);
     invoker.invokeTeardown();
     verify(fn).before();
-    verify(fn).startBundle(mockProcessContext);
-    verify(fn).finishBundle(mockProcessContext);
+    verify(fn).startBundle(mockStartBundleContext);
+    verify(fn).finishBundle(mockFinishBundleContext);
     verify(fn).after();
   }
 
@@ -601,7 +607,7 @@ public void testStartBundleException() throws Exception {
         DoFnInvokers.invokerFor(
             new DoFn<Integer, Integer>() {
               @StartBundle
-              public void startBundle(@SuppressWarnings("unused") Context c) {
+              public void startBundle(@SuppressWarnings("unused") StartBundleContext c) {
                 throw new IllegalArgumentException("bogus");
               }
 
@@ -619,7 +625,7 @@ public void testFinishBundleException() throws Exception {
         DoFnInvokers.invokerFor(
             new DoFn<Integer, Integer>() {
               @FinishBundle
-              public void finishBundle(@SuppressWarnings("unused") Context c) {
+              public void finishBundle(@SuppressWarnings("unused") FinishBundleContext c) {
                 throw new IllegalArgumentException("bogus");
               }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
index d6cc4f6a51f1..f099d5d1b846 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -76,13 +76,14 @@ public void process(ProcessContext c) {}
   @Test
   public void testBadExtraContext() throws Exception {
     thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("Must take a single argument of type DoFn<Integer, String>.Context");
+    thrown.expectMessage(
+        "Must take a single argument of type DoFn<Integer, String>.StartBundleContext");
 
-    DoFnSignatures.analyzeBundleMethod(
+    DoFnSignatures.analyzeStartBundleMethod(
         errors(),
         TypeDescriptor.of(FakeDoFn.class),
         new DoFnSignaturesTestUtils.AnonymousMethod() {
-          void method(DoFn<Integer, String>.Context c, int n) {}
+          void method(DoFn<Integer, String>.StartBundleContext c, int n) {}
         }.getMethod(),
         TypeDescriptor.of(Integer.class),
         TypeDescriptor.of(String.class));
@@ -112,8 +113,8 @@ public void baz() {}
   public void testMultipleFinishBundleMethods() throws Exception {
     thrown.expect(IllegalArgumentException.class);
     thrown.expectMessage("Found multiple methods annotated with @FinishBundle");
-    thrown.expectMessage("bar(Context)");
-    thrown.expectMessage("baz(Context)");
+    thrown.expectMessage("bar(FinishBundleContext)");
+    thrown.expectMessage("baz(FinishBundleContext)");
     thrown.expectMessage(getClass().getName() + "$");
     DoFnSignatures.getSignature(
         new DoFn<String, String>() {
@@ -121,10 +122,10 @@ public void testMultipleFinishBundleMethods() throws Exception {
           public void foo(ProcessContext context) {}
 
           @FinishBundle
-          public void bar(Context context) {}
+          public void bar(FinishBundleContext context) {}
 
           @FinishBundle
-          public void baz(Context context) {}
+          public void baz(FinishBundleContext context) {}
         }.getClass());
   }
 
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 1cdd087cfd47..6d5e230b9c20 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -71,6 +71,7 @@
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -304,20 +305,21 @@ private static class TestDoFn extends DoFn<String, String> {
     private static final TupleTag<String> mainOutput = new TupleTag<>("mainOutput");
     private static final TupleTag<String> additionalOutput = new TupleTag<>("output");
 
-    @StartBundle
-    public void startBundle(Context context) {
-      context.output("StartBundle");
-    }
+    private BoundedWindow window;
 
     @ProcessElement
-    public void processElement(ProcessContext context) {
+    public void processElement(ProcessContext context, BoundedWindow window) {
       context.output("MainOutput" + context.element());
       context.output(additionalOutput, "AdditionalOutput" + context.element());
+      this.window = window;
     }
 
     @FinishBundle
-    public void finishBundle(Context context) {
-      context.output("FinishBundle");
+    public void finishBundle(FinishBundleContext context) {
+      if (window != null) {
+        context.output("FinishBundle", window.maxTimestamp(), window);
+        window = null;
+      }
     }
   }
 
@@ -411,7 +413,6 @@ public void testCreatingAndProcessingDoFn() throws Exception {
         finishFunctions::add);
 
     Iterables.getOnlyElement(startFunctions).run();
-    assertThat(mainOutputValues, contains(valueInGlobalWindow("StartBundle")));
     mainOutputValues.clear();
 
     assertEquals(newConsumers.keySet(),
diff --git a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
index 0a3b900c548b..f6ceef2286f5 100644
--- a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
+++ b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
@@ -25,7 +25,6 @@
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
-
 import java.io.IOException;
 import java.io.Serializable;
 import java.net.MalformedURLException;
@@ -40,7 +39,6 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -757,7 +755,7 @@ public void createClient() throws Exception {
       }
 
       @StartBundle
-      public void startBundle(Context context) throws Exception {
+      public void startBundle(StartBundleContext context) throws Exception {
         batch = new ArrayList<>();
         currentBatchSizeBytes = 0;
       }
@@ -769,12 +767,16 @@ public void processElement(ProcessContext context) throws Exception {
         currentBatchSizeBytes += document.getBytes().length;
         if (batch.size() >= spec.getMaxBatchSize()
             || currentBatchSizeBytes >= spec.getMaxBatchSizeBytes()) {
-          finishBundle(context);
+          flushBatch();
         }
       }
 
       @FinishBundle
-      public void finishBundle(Context context) throws Exception {
+      public void finishBundle(FinishBundleContext context) throws Exception {
+        flushBatch();
+      }
+
+      private void flushBatch() throws IOException {
         if (batch.isEmpty()) {
           return;
         }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
index fd5f39655cb5..f267976f68dd 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
@@ -56,7 +56,7 @@ class StreamingWriteFn
 
   /** Prepares a target BigQuery table. */
   @StartBundle
-  public void startBundle(Context context) {
+  public void startBundle() {
     tableRows = new HashMap<>();
     uniqueIdsForTableRows = new HashMap<>();
   }
@@ -75,7 +75,7 @@ public void processElement(ProcessContext context) {
 
   /** Writes the accumulated rows into BigQuery with streaming API. */
   @FinishBundle
-  public void finishBundle(Context context) throws Exception {
+  public void finishBundle(FinishBundleContext context) throws Exception {
     BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class);
     for (Map.Entry<String, List<TableRow>> entry : tableRows.entrySet()) {
       TableReference tableReference = BigQueryHelpers.parseTableSpec(entry.getKey());
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java
index 284691e49f55..cd88222da69f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java
@@ -40,7 +40,7 @@ class TagWithUniqueIds
   private transient long sequenceNo = 0L;
 
   @StartBundle
-  public void startBundle(Context context) {
+  public void startBundle() {
     randomUUID = UUID.randomUUID().toString();
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
index e90b974d973f..70aa135aab27 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
@@ -34,6 +34,7 @@
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.KV;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -48,8 +49,10 @@ class WriteBundlesToFiles<DestinationT>
 
   // Map from tablespec to a writer for that table.
   private transient Map<DestinationT, TableRowWriter> writers;
+  private transient Map<DestinationT, BoundedWindow> writerWindows;
   private final String stepUuid;
 
+
   /**
    * The result of the {@link WriteBundlesToFiles} transform. Corresponds to a single output file,
    * and encapsulates the table it is destined to as well as the file byte size.
@@ -110,14 +113,15 @@ public void verifyDeterministic() {}
   }
 
   @StartBundle
-  public void startBundle(Context c) {
+  public void startBundle() {
     // This must be done each bundle, as by default the {@link DoFn} might be reused between
     // bundles.
     this.writers = Maps.newHashMap();
+    this.writerWindows = Maps.newHashMap();
   }
 
   @ProcessElement
-  public void processElement(ProcessContext c) throws Exception {
+  public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
     String tempFilePrefix = resolveTempLocation(
         c.getPipelineOptions().getTempLocation(), "BigQueryWriteTemp", stepUuid);
     TableRowWriter writer = writers.get(c.element().getKey());
@@ -125,6 +129,7 @@ public void processElement(ProcessContext c) throws Exception {
       writer = new TableRowWriter(tempFilePrefix);
       writer.open(UUID.randomUUID().toString());
       writers.put(c.element().getKey(), writer);
+      writerWindows.put(c.element().getKey(), window);
       LOG.debug("Done opening writer {}", writer);
     }
     try {
@@ -143,11 +148,15 @@ public void processElement(ProcessContext c) throws Exception {
   }
 
   @FinishBundle
-  public void finishBundle(Context c) throws Exception {
+  public void finishBundle(FinishBundleContext c) throws Exception {
     for (Map.Entry<DestinationT, TableRowWriter> entry : writers.entrySet()) {
       TableRowWriter.Result result = entry.getValue().close();
-      c.output(new Result<>(result.resourceId.toString(), result.byteSize, entry.getKey()));
+      c.output(
+          new Result<>(result.resourceId.toString(), result.byteSize, entry.getKey()),
+          writerWindows.get(entry.getKey()).maxTimestamp(),
+          writerWindows.get(entry.getKey()));
     }
     writers.clear();
+    writerWindows.clear();
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 69fac6836329..0e97c1254d93 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -572,7 +572,7 @@ public BigtableWriterFn(String tableId,
       }
 
       @StartBundle
-      public void startBundle(Context c) throws IOException {
+      public void startBundle(StartBundleContext c) throws IOException {
         if (bigtableWriter == null) {
           bigtableWriter = bigtableServiceFactory.apply(
               c.getPipelineOptions()).openForWriting(tableId);
@@ -589,7 +589,7 @@ public void processElement(ProcessContext c) throws Exception {
       }
 
       @FinishBundle
-      public void finishBundle(Context c) throws Exception {
+      public void finishBundle() throws Exception {
         bigtableWriter.flush();
         checkForFailures();
         LOG.info("Wrote {} records", recordsWritten);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index f6194296049a..fd4fccf0ba1d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -704,7 +704,7 @@ static class GqlQueryTranslateFn extends DoFn<ValueProvider<String>, Query> {
       }
 
       @StartBundle
-      public void startBundle(Context c) throws Exception {
+      public void startBundle(StartBundleContext c) throws Exception {
         datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), v1Options.getProjectId());
       }
 
@@ -748,7 +748,7 @@ public SplitQueryFn(V1Options options, int numSplits) {
       }
 
       @StartBundle
-      public void startBundle(Context c) throws Exception {
+      public void startBundle(StartBundleContext c) throws Exception {
         datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), options.getProjectId(),
             options.getLocalhost());
         querySplitter = datastoreFactory.getQuerySplitter();
@@ -821,7 +821,7 @@ public ReadFn(V1Options options) {
       }
 
       @StartBundle
-      public void startBundle(Context c) throws Exception {
+      public void startBundle(StartBundleContext c) throws Exception {
         datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), options.getProjectId(),
             options.getLocalhost());
       }
@@ -1145,7 +1145,7 @@ static class DatastoreWriterFn extends DoFn<Mutation, Void> {
     }
 
     @StartBundle
-    public void startBundle(Context c) {
+    public void startBundle(StartBundleContext c) {
       datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), projectId.get(), localhost);
     }
 
@@ -1158,7 +1158,7 @@ public void processElement(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
+    public void finishBundle() throws Exception {
       if (!mutations.isEmpty()) {
         flushBatch();
       }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index e023ad0dca07..fa2d20f317ba 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -888,7 +888,7 @@ public class PubsubBoundedWriter extends DoFn<T, Void> {
       private transient PubsubClient pubsubClient;
 
       @StartBundle
-      public void startBundle(Context c) throws IOException {
+      public void startBundle(StartBundleContext c) throws IOException {
         this.output = new ArrayList<>();
         // NOTE: idAttribute is ignored.
         this.pubsubClient =
@@ -911,7 +911,7 @@ public void processElement(ProcessContext c) throws IOException {
       }
 
       @FinishBundle
-      public void finishBundle(Context c) throws IOException {
+      public void finishBundle() throws IOException {
         if (!output.isEmpty()) {
           publish();
         }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
index 9d97e91f50bc..031d9a0ab14e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
@@ -255,7 +255,7 @@ private void publishBatch(List<OutgoingMessage> messages, int bytes)
     }
 
     @StartBundle
-    public void startBundle(Context c) throws Exception {
+    public void startBundle(StartBundleContext c) throws Exception {
       checkState(pubsubClient == null, "startBundle invoked without prior finishBundle");
       pubsubClient = pubsubFactory.newClient(timestampAttribute, idAttribute,
                                              c.getPipelineOptions().as(PubsubOptions.class));
@@ -287,7 +287,7 @@ public void processElement(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
+    public void finishBundle() throws Exception {
       pubsubClient.close();
       pubsubClient = null;
     }
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java
index 86a9246a3516..ef6556eb481d 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java
@@ -223,6 +223,7 @@ private class WriteBundles<WriteT> extends DoFn<T, WriteT> {
     // Writer that will write the records in this bundle. Lazily
     // initialized in processElement.
     private Writer<T, WriteT> writer = null;
+    private BoundedWindow window;
     private final PCollectionView<WriteOperation<T, WriteT>> writeOperationView;
 
     WriteBundles(PCollectionView<WriteOperation<T, WriteT>> writeOperationView) {
@@ -243,6 +244,7 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
         } else {
           writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, UNKNOWN_NUMSHARDS);
         }
+        this.window = window;
         LOG.debug("Done opening writer {} for operation {}", writer, writeOperationView);
       }
       try {
@@ -265,12 +267,13 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
+    public void finishBundle(FinishBundleContext c) throws Exception {
       if (writer != null) {
         WriteT result = writer.close();
-        c.output(result);
+        c.output(result, window.maxTimestamp(), window);
         // Reset state in case of reuse.
         writer = null;
+        window = null;
       }
     }
 
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
index eee892701123..3c42da91f27a 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
@@ -634,11 +634,6 @@ public void setup() throws Exception {
                 recordsWritten = 0;
             }
 
-            @StartBundle
-            public void startBundle(Context c) throws Exception {
-
-            }
-
             @ProcessElement
             public void processElement(ProcessContext ctx) throws Exception {
                 KV<byte[], Iterable<Mutation>> record = ctx.element();
@@ -651,7 +646,7 @@ public void processElement(ProcessContext ctx) throws Exception {
             }
 
             @FinishBundle
-            public void finishBundle(Context c) throws Exception {
+            public void finishBundle() throws Exception {
                 mutator.flush();
             }
 
diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
index 2d4823679fac..2eb53dd6361a 100644
--- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
+++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
@@ -25,6 +25,7 @@
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.Random;
 
 import javax.annotation.Nullable;
@@ -480,7 +481,7 @@ public void setup() throws Exception {
       }
 
       @StartBundle
-      public void startBundle(Context context) {
+      public void startBundle() {
         batchCount = 0;
       }
 
@@ -495,12 +496,16 @@ public void processElement(ProcessContext context) throws Exception {
         batchCount++;
 
         if (batchCount >= DEFAULT_BATCH_SIZE) {
-          finishBundle(context);
+          executeBatch();
         }
       }
 
       @FinishBundle
-      public void finishBundle(Context context) throws Exception {
+      public void finishBundle() throws Exception {
+        executeBatch();
+      }
+
+      private void executeBatch() throws SQLException {
         if (batchCount > 0) {
           preparedStatement.executeBatch();
           connection.commit();
diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
index 813e0512d44f..4493e56d3fee 100644
--- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
+++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
@@ -674,7 +674,7 @@ public WriterFn(Write spec) {
       }
 
       @StartBundle
-      public void startBundle(Context c) throws Exception {
+      public void startBundle() throws Exception {
         if (producer == null) {
           if (spec.getUsername() != null) {
             this.connection =
@@ -703,13 +703,13 @@ public void processElement(ProcessContext ctx) throws Exception {
           TextMessage message = session.createTextMessage(value);
           producer.send(message);
         } catch (Exception t) {
-          finishBundle(null);
+          finishBundle();
           throw t;
         }
       }
 
       @FinishBundle
-      public void finishBundle(Context c) throws Exception {
+      public void finishBundle() throws Exception {
         producer.close();
         producer = null;
         session.close();
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index 8ab33d1c5b69..f4de76a56b47 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -1632,7 +1632,7 @@ public void processElement(ProcessContext ctx) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws IOException {
+    public void finishBundle() throws IOException {
       producer.flush();
       checkForFailures();
     }
diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
index 940d87534ac9..0868ed443c84 100644
--- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
+++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
@@ -641,7 +641,7 @@ public void setup() throws Exception {
     }
 
     @StartBundle
-    public void startBundle(Context context) {
+    public void startBundle() {
       gridFsFile = gridfs.createFile(spec.filename());
       if (spec.chunkSize() != null) {
         gridFsFile.setChunkSize(spec.chunkSize());
@@ -656,7 +656,7 @@ public void processElement(ProcessContext context) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context context) throws Exception {
+    public void finishBundle() throws Exception {
       if (gridFsFile != null) {
         outputStream.flush();
         outputStream.close();
diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
index f8edbf11a8e9..7236a50655e5 100644
--- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
+++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
@@ -466,7 +466,7 @@ public void createMongoClient() throws Exception {
       }
 
       @StartBundle
-      public void startBundle(Context ctx) throws Exception {
+      public void startBundle() throws Exception {
         batch = new ArrayList<>();
       }
 
@@ -476,12 +476,16 @@ public void processElement(ProcessContext ctx) throws Exception {
         // before inserting (will assign an id).
         batch.add(new Document(ctx.element()));
         if (batch.size() >= spec.batchSize()) {
-          finishBundle(ctx);
+          flush();
         }
       }
 
       @FinishBundle
-      public void finishBundle(Context ctx) throws Exception {
+      public void finishBundle() throws Exception {
+        flush();
+      }
+
+      private void flush() {
         MongoDatabase mongoDatabase = client.getDatabase(spec.database());
         MongoCollection<Document> mongoCollection = mongoDatabase.getCollection(spec.collection());
 

From d9943a3cbd402872053f5482cf08cb3b70416bd4 Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Thu, 4 May 2017 14:12:24 -0700
Subject: [PATCH 047/387] [BEAM-2170] PubsubMessageWithAttributesCoder should
 not NPE on messages without attributes

---
 .../PubsubMessageWithAttributesCoder.java     | 14 ++++----
 .../gcp/pubsub/PubsubUnboundedSinkTest.java   | 34 ++++++++++++++++---
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
index f70955da92e0..e061edc50106 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
@@ -31,10 +31,11 @@
 
 /** A coder for PubsubMessage including attributes. */
 public class PubsubMessageWithAttributesCoder extends CustomCoder<PubsubMessage> {
-  private static final Coder<byte[]> PAYLOAD_CODER =
-      NullableCoder.of(ByteArrayCoder.of());
-  private static final Coder<Map<String, String>> ATTRIBUTES_CODER = MapCoder.of(
-      StringUtf8Coder.of(), StringUtf8Coder.of());
+  // A message's payload can not be null
+  private static final Coder<byte[]> PAYLOAD_CODER = ByteArrayCoder.of();
+  // A message's attributes can be null.
+  private static final Coder<Map<String, String>> ATTRIBUTES_CODER =
+      NullableCoder.of(MapCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()));
 
   public static Coder<PubsubMessage> of(TypeDescriptor<PubsubMessage> ignored) {
     return of();
@@ -46,10 +47,7 @@ public static PubsubMessageWithAttributesCoder of() {
 
   public void encode(PubsubMessage value, OutputStream outStream, Context context)
       throws IOException {
-    PAYLOAD_CODER.encode(
-        value.getPayload(),
-        outStream,
-        context.nested());
+    PAYLOAD_CODER.encode(value.getPayload(), outStream, context.nested());
     ATTRIBUTES_CODER.encode(value.getAttributeMap(), outStream, context);
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java
index cc3c85e17ff2..e32e9a86f400 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java
@@ -33,7 +33,6 @@
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubUnboundedSink.RecordIdMethod;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.testing.CoderProperties;
-import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -42,7 +41,6 @@
 import org.joda.time.Instant;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -96,7 +94,6 @@ public void saneCoder() throws Exception {
   }
 
   @Test
-  @Category(NeedsRunner.class)
   public void sendOneMessage() throws IOException {
     List<OutgoingMessage> outgoing =
         ImmutableList.of(new OutgoingMessage(
@@ -123,7 +120,35 @@ public void sendOneMessage() throws IOException {
   }
 
   @Test
-  @Category(NeedsRunner.class)
+  public void sendOneMessageWithoutAttributes() throws IOException {
+    List<OutgoingMessage> outgoing =
+        ImmutableList.of(
+            new OutgoingMessage(
+                DATA.getBytes(), null /* attributes */, TIMESTAMP, getRecordId(DATA)));
+    try (PubsubTestClientFactory factory =
+        PubsubTestClient.createFactoryForPublish(
+            TOPIC, outgoing, ImmutableList.<OutgoingMessage>of())) {
+      PubsubUnboundedSink sink =
+          new PubsubUnboundedSink(
+              factory,
+              StaticValueProvider.of(TOPIC),
+              TIMESTAMP_ATTRIBUTE,
+              ID_ATTRIBUTE,
+              NUM_SHARDS,
+              1 /* batchSize */,
+              1 /* batchBytes */,
+              Duration.standardSeconds(2),
+              RecordIdMethod.DETERMINISTIC);
+      p.apply(Create.of(ImmutableList.of(DATA)))
+          .apply(ParDo.of(new Stamp(null /* attributes */)))
+          .apply(sink);
+      p.run();
+    }
+    // The PubsubTestClientFactory will assert fail on close if the actual published
+    // message does not match the expected publish message.
+  }
+
+  @Test
   public void sendMoreThanOneBatchByNumMessages() throws IOException {
     List<OutgoingMessage> outgoing = new ArrayList<>();
     List<String> data = new ArrayList<>();
@@ -152,7 +177,6 @@ public void sendMoreThanOneBatchByNumMessages() throws IOException {
   }
 
   @Test
-  @Category(NeedsRunner.class)
   public void sendMoreThanOneBatchByByteSize() throws IOException {
     List<OutgoingMessage> outgoing = new ArrayList<>();
     List<String> data = new ArrayList<>();

From c9b7fe443368badf6fd9fbd08f5234f17766c2cf Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Thu, 4 May 2017 14:40:08 -0700
Subject: [PATCH 048/387] Makes PubsubMessagePayloadOnlyCoder not require
 whole-stream context

Now that PubsubIO.Read can directly read PubsubMessage's,
they should be treated as first-class PCollection elements, and
they can be encoded/decoded in any contexts.
---
 .../gcp/pubsub/PubsubMessagePayloadOnlyCoder.java   | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java
index 81c1a45b4d5b..d120f7225fe5 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java
@@ -17,17 +17,18 @@
  */
 package org.apache.beam.sdk.io.gcp.pubsub;
 
-import static com.google.common.base.Preconditions.checkState;
-
 import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CustomCoder;
-import org.apache.beam.sdk.util.StreamUtils;
 
 /** A coder for PubsubMessage treating the raw bytes being decoded as the message's payload. */
 public class PubsubMessagePayloadOnlyCoder extends CustomCoder<PubsubMessage> {
+  private static final Coder<byte[]> PAYLOAD_CODER = ByteArrayCoder.of();
+
   public static PubsubMessagePayloadOnlyCoder of() {
     return new PubsubMessagePayloadOnlyCoder();
   }
@@ -35,14 +36,12 @@ public static PubsubMessagePayloadOnlyCoder of() {
   @Override
   public void encode(PubsubMessage value, OutputStream outStream, Context context)
       throws IOException {
-    checkState(context.isWholeStream, "Expected to only be used in a whole-stream context");
-    outStream.write(value.getPayload());
+    PAYLOAD_CODER.encode(value.getPayload(), outStream, context);
   }
 
   @Override
   public PubsubMessage decode(InputStream inStream, Context context) throws IOException {
-    checkState(context.isWholeStream, "Expected to only be used in a whole-stream context");
     return new PubsubMessage(
-        StreamUtils.getBytes(inStream), ImmutableMap.<String, String>of());
+        PAYLOAD_CODER.decode(inStream, context), ImmutableMap.<String, String>of());
   }
 }

From 9d152498414ead3db77d64f49a2e90f4d4d255e0 Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Thu, 4 May 2017 14:58:40 -0700
Subject: [PATCH 049/387] Moves coder choice into PubsubSource

---
 .../java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 9 +--------
 .../beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java    | 8 ++++----
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index fa2d20f317ba..51da111ccd24 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -721,14 +721,7 @@ public PCollection<T> expand(PBegin input) {
               getTimestampAttribute(),
               getIdAttribute(),
               getNeedsAttributes());
-      return input
-          .getPipeline()
-          .apply(source)
-          .setCoder(
-              getNeedsAttributes()
-                  ? PubsubMessageWithAttributesCoder.of()
-                  : PubsubMessagePayloadOnlyCoder.of())
-          .apply(MapElements.via(getParseFn()));
+      return input.apply(source).apply(MapElements.via(getParseFn()));
     }
 
     @Override
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index e5be71ba9e29..c2cbe73f484b 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -1155,14 +1155,14 @@ public PubsubReader createReader(
     @Nullable
     @Override
     public Coder<PubsubCheckpoint> getCheckpointMarkCoder() {
-      @SuppressWarnings("unchecked") PubsubCheckpointCoder typedCoder =
-          (PubsubCheckpointCoder) CHECKPOINT_CODER;
-      return typedCoder;
+      return CHECKPOINT_CODER;
     }
 
     @Override
     public Coder<PubsubMessage> getDefaultOutputCoder() {
-      return new PubsubMessageWithAttributesCoder();
+      return outer.getNeedsAttributes()
+          ? PubsubMessageWithAttributesCoder.of()
+          : PubsubMessagePayloadOnlyCoder.of();
     }
 
     @Override

From 78e0accad907420675aefd0ac617bcf5d2a82619 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 10:29:33 -0700
Subject: [PATCH 050/387] Move Java sdk.util.state to sdk.state

---
 .../translation/utils/ApexStateInternals.java | 20 ++++++++---------
 .../utils/StateInternalsProxy.java            |  4 ++--
 .../utils/ApexStateInternalsTest.java         | 12 +++++-----
 .../construction/PTransformMatchersTest.java  |  6 ++---
 .../beam/runners/core/DoFnAdapters.java       |  2 +-
 .../runners/core/InMemoryStateInternals.java  | 22 +++++++++----------
 .../runners/core/MergingActiveWindowSet.java  |  2 +-
 .../runners/core/MergingStateAccessor.java    |  2 +-
 .../beam/runners/core/NonEmptyPanes.java      |  4 ++--
 ...oundedSplittableProcessElementInvoker.java |  2 +-
 .../beam/runners/core/PaneInfoTracker.java    |  4 ++--
 .../apache/beam/runners/core/ReduceFn.java    |  2 +-
 .../runners/core/ReduceFnContextFactory.java  |  6 ++---
 .../beam/runners/core/SideInputHandler.java   |  4 ++--
 .../beam/runners/core/SimpleDoFnRunner.java   |  4 ++--
 .../beam/runners/core/SplittableParDo.java    |  4 ++--
 .../beam/runners/core/StateAccessor.java      |  2 +-
 .../beam/runners/core/StateInternals.java     |  4 ++--
 .../beam/runners/core/StateMerging.java       | 14 ++++++------
 .../apache/beam/runners/core/StateTable.java  |  4 ++--
 .../apache/beam/runners/core/StateTag.java    | 16 +++++++-------
 .../apache/beam/runners/core/StateTags.java   | 20 ++++++++---------
 .../beam/runners/core/StatefulDoFnRunner.java |  4 ++--
 .../beam/runners/core/SystemReduceFn.java     |  8 +++----
 .../core/TestInMemoryStateInternals.java      |  4 ++--
 .../beam/runners/core/WatermarkHold.java      |  4 ++--
 ...fterDelayFromFirstElementStateMachine.java |  4 ++--
 .../core/triggers/AfterPaneStateMachine.java  |  2 +-
 .../TriggerStateMachineContextFactory.java    |  2 +-
 .../triggers/TriggerStateMachineRunner.java   |  2 +-
 .../core/InMemoryStateInternalsTest.java      | 16 +++++++-------
 .../runners/core/StatefulDoFnRunnerTest.java  |  6 ++---
 .../CopyOnAccessInMemoryStateInternals.java   | 18 +++++++--------
 .../direct/StatefulParDoEvaluatorFactory.java |  2 +-
 ...opyOnAccessInMemoryStateInternalsTest.java | 14 ++++++------
 .../runners/direct/EvaluationContextTest.java |  2 +-
 .../StatefulParDoEvaluatorFactoryTest.java    |  6 ++---
 .../flink/FlinkDetachedRunnerResult.java      |  1 -
 .../types/EncodedValueSerializer.java         |  2 --
 .../wrappers/streaming/DoFnOperator.java      |  2 +-
 .../state/FlinkBroadcastStateInternals.java   | 20 ++++++++---------
 .../state/FlinkKeyGroupStateInternals.java    | 20 ++++++++---------
 .../state/FlinkSplitStateInternals.java       | 20 ++++++++---------
 .../streaming/state/FlinkStateInternals.java  | 20 ++++++++---------
 .../flink/streaming/DoFnOperatorTest.java     |  6 ++---
 .../FlinkBroadcastStateInternalsTest.java     | 10 ++++-----
 .../FlinkKeyGroupStateInternalsTest.java      |  4 ++--
 .../FlinkSplitStateInternalsTest.java         |  4 ++--
 .../streaming/FlinkStateInternalsTest.java    | 12 +++++-----
 .../BatchStatefulParDoOverridesTest.java      |  6 ++---
 .../DataflowPipelineTranslatorTest.java       |  6 ++---
 .../spark/stateful/SparkStateInternals.java   | 20 ++++++++---------
 .../main/resources/beam/findbugs-filter.xml   |  2 +-
 .../beam/sdk/{util => }/state/BagState.java   |  2 +-
 .../sdk/{util => }/state/CombiningState.java  |  2 +-
 .../sdk/{util => }/state/GroupingState.java   |  2 +-
 .../beam/sdk/{util => }/state/MapState.java   |  2 +-
 .../sdk/{util => }/state/ReadableState.java   |  2 +-
 .../sdk/{util => }/state/ReadableStates.java  |  2 +-
 .../beam/sdk/{util => }/state/SetState.java   |  2 +-
 .../beam/sdk/{util => }/state/State.java      |  2 +-
 .../sdk/{util => }/state/StateBinder.java     |  2 +-
 .../sdk/{util => }/state/StateContext.java    |  2 +-
 .../sdk/{util => }/state/StateContexts.java   |  2 +-
 .../beam/sdk/{util => }/state/StateSpec.java  |  2 +-
 .../beam/sdk/{util => }/state/StateSpecs.java |  2 +-
 .../beam/sdk/{util => }/state/ValueState.java |  2 +-
 .../{util => }/state/WatermarkHoldState.java  |  2 +-
 .../sdk/{util => }/state/package-info.java    |  2 +-
 .../apache/beam/sdk/testing/UsesMapState.java |  2 +-
 .../apache/beam/sdk/testing/UsesSetState.java |  2 +-
 .../org/apache/beam/sdk/transforms/DoFn.java  |  4 ++--
 .../beam/sdk/transforms/DoFnTester.java       |  2 +-
 .../beam/sdk/transforms/GroupIntoBatches.java | 10 ++++-----
 .../org/apache/beam/sdk/transforms/ParDo.java |  2 +-
 .../sdk/transforms/reflect/DoFnInvoker.java   |  2 +-
 .../sdk/transforms/reflect/DoFnSignature.java |  4 ++--
 .../transforms/reflect/DoFnSignatures.java    |  4 ++--
 .../beam/sdk/util/CombineContextFactory.java  |  2 +-
 .../apache/beam/sdk/util/CombineFnUtil.java   |  2 +-
 .../apache/beam/sdk/transforms/ParDoTest.java | 14 ++++++------
 .../transforms/reflect/DoFnInvokersTest.java  |  6 ++---
 .../reflect/DoFnSignaturesTest.java           |  8 +++----
 .../beam/sdk/util/CombineFnUtilTest.java      |  2 +-
 84 files changed, 252 insertions(+), 255 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/BagState.java (96%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/CombiningState.java (98%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/GroupingState.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/MapState.java (98%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/ReadableState.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/ReadableStates.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/SetState.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/State.java (96%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/StateBinder.java (98%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/StateContext.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/StateContexts.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/StateSpec.java (98%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/StateSpecs.java (99%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/ValueState.java (96%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/WatermarkHoldState.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => }/state/package-info.java (95%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
index 430056718696..eeea6d144474 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
@@ -41,22 +41,22 @@
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.sdk.util.CombineFnUtil;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java
index 746be2f8f572..ccf7e43e536c 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java
@@ -24,8 +24,8 @@
 import org.apache.beam.runners.core.StateInternalsFactory;
 import org.apache.beam.runners.core.StateNamespace;
 import org.apache.beam.runners.core.StateTag;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
 
 /**
  * State internals for reusable processing context.
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java
index 8b48a74148b8..a7e64af4dd5b 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternalsTest.java
@@ -33,16 +33,16 @@
 import org.apache.beam.runners.core.StateTags;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index bb1b1cd63184..278c12bf8bcf 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -37,6 +37,9 @@
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.runners.PTransformMatcher;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
@@ -60,9 +63,6 @@
 import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
index fd1772ad2761..849400f1d06d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import org.apache.beam.runners.core.OldDoFn.Context;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
@@ -34,7 +35,6 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.state.State;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.joda.time.Duration;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java
index 199ce41a867d..59814bc754c2 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java
@@ -30,22 +30,22 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.ReadableStates;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CombineFnUtil;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.ReadableStates;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java
index 2faedbb0f630..185892eca81b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java
@@ -36,9 +36,9 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.MapCoder;
 import org.apache.beam.sdk.coders.SetCoder;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.state.ValueState;
 
 /**
  * An {@link ActiveWindowSet} for merging {@link WindowFn} implementations.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingStateAccessor.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingStateAccessor.java
index 5ffb9a22218a..361e8c527d3d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingStateAccessor.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingStateAccessor.java
@@ -20,8 +20,8 @@
 import java.util.Map;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.state.State;
 
 /**
  * Interface for accessing persistent state while windows are merging.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java
index 06dcc9cdcfe1..8df7ad005aa4 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java
@@ -18,12 +18,12 @@
 package org.apache.beam.runners.core;
 
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.ReadableState;
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.ReadableState;
 
 /**
  * Tracks which windows have non-empty panes. Specifically, which windows have new elements since
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index a51041ed9a99..8e80a694d437 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -27,6 +27,7 @@
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
@@ -37,7 +38,6 @@
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.state.State;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.joda.time.Duration;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java
index 66b39608a4af..24a4576204fd 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java
@@ -20,13 +20,13 @@
 import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.PaneInfoCoder;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
 import org.apache.beam.sdk.util.WindowTracing;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
index cd62d859aa54..bcc47a84600b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
@@ -18,11 +18,11 @@
 package org.apache.beam.runners.core;
 
 import java.io.Serializable;
+import org.apache.beam.sdk.state.ReadableState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.Timers;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.ReadableState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
index 3031ebf6d8f7..cb1a15964e32 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
@@ -28,15 +28,15 @@
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.Timers;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
index b29f9d0281bb..af750104e6dc 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
@@ -28,12 +28,12 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.SetCoder;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 768378daceca..200a43b5952f 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -32,6 +32,8 @@
 import org.apache.beam.runners.core.ExecutionContext.StepContext;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
@@ -55,8 +57,6 @@
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateSpec;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.joda.time.Duration;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
index 7cf5e88a4c99..1fda7d92a176 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
@@ -29,6 +29,8 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -47,8 +49,6 @@
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateAccessor.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateAccessor.java
index eda896bc683f..6c4052023586 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateAccessor.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateAccessor.java
@@ -19,7 +19,7 @@
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.util.state.State;
+import org.apache.beam.sdk.state.State;
 
 /**
  * Interface for accessing a {@link StateTag} in the current context.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateInternals.java
index c2e94120c707..abf961dea8a4 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateInternals.java
@@ -19,9 +19,9 @@
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
 import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
 
 /**
  * {@code StateInternals} describes the functionality a runner needs to provide for the
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateMerging.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateMerging.java
index f6b91032e5d7..8285c7245d25 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateMerging.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateMerging.java
@@ -23,14 +23,14 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java
index 1bf4ff511392..d996729a476a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java
@@ -22,8 +22,8 @@
 import java.util.Map;
 import java.util.Set;
 import org.apache.beam.runners.core.StateTag.StateBinder;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
 
 /**
  * Table mapping {@code StateNamespace} and {@code StateTag<?>} to a {@code State} instance.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java
index 38e9dea6f712..f7c309e208a1 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTag.java
@@ -22,18 +22,18 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 
 /**
  * An address and specification for a persistent state cell. This includes a unique identifier for
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java
index ca8b2387f52e..53f9edcc9f4b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java
@@ -25,20 +25,20 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateBinder;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateBinder;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 
 /**
  * Static utility methods for creating {@link StateTag} instances.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
index e3717a89daeb..0a7701aed8d9 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
@@ -21,6 +21,8 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
@@ -31,8 +33,6 @@
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateSpec;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java
index f18460ae4044..c189b0d7ee3f 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java
@@ -19,15 +19,15 @@
 
 
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.ReadableState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.AppliedCombineFn;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
-import org.apache.beam.sdk.util.state.ReadableState;
 
 /**
  * {@link ReduceFn} implementing the default reduction behaviors of {@link GroupByKey}.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java
index 18b50dba0b39..2052c039f80a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TestInMemoryStateInternals.java
@@ -20,8 +20,8 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
index e6e4ffb3bafe..ce15244d795c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
@@ -22,14 +22,14 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.io.Serializable;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
index ed2c26f68a54..b972985fa3ac 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
@@ -30,12 +30,12 @@
 import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.InstantCoder;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
 import org.apache.beam.sdk.transforms.Combine.Holder;
 import org.apache.beam.sdk.transforms.Min;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.joda.time.format.PeriodFormat;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java
index 52fb5ff495e3..b9fbac34d0fb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterPaneStateMachine.java
@@ -26,8 +26,8 @@
 import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.state.CombiningState;
 import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.util.state.CombiningState;
 
 /**
  * {@link TriggerStateMachine}s that fire based on properties of the elements in the current pane.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java
index a0569374b396..84b0453d12fc 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java
@@ -34,11 +34,11 @@
 import org.apache.beam.runners.core.triggers.TriggerStateMachine.MergingTriggerInfo;
 import org.apache.beam.runners.core.triggers.TriggerStateMachine.TriggerInfo;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.Timers;
-import org.apache.beam.sdk.util.state.State;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
index fc2f69681480..324d44d7d515 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
@@ -29,9 +29,9 @@
 import org.apache.beam.runners.core.StateTag;
 import org.apache.beam.runners.core.StateTags;
 import org.apache.beam.sdk.coders.BitSetCoder;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.Timers;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java
index 16f7f26dc965..b526305cea00 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryStateInternalsTest.java
@@ -32,18 +32,18 @@
 import java.util.Objects;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;
 import org.junit.Test;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index d4ff49e0b849..d7d8261cca5f 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -30,6 +30,9 @@
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
@@ -38,9 +41,6 @@
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
 import org.joda.time.Duration;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java
index d2af93cb5a32..3c701c77695a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java
@@ -39,20 +39,20 @@
 import org.apache.beam.runners.core.StateTag;
 import org.apache.beam.runners.core.StateTag.StateBinder;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CombineFnUtil;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index f278e08ba98f..ca302b8b11a6 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -39,6 +39,7 @@
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.ParDoMultiOverrideFactory.StatefulParDo;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -48,7 +49,6 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.StateSpec;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
index 3e29a69bdf82..46f26a1d4701 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
@@ -38,18 +38,18 @@
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.joda.time.Instant;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 40582d9805d3..3eff11a3436a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -43,6 +43,7 @@
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.state.BagState;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
@@ -56,7 +57,6 @@
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.BagState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
index 67f2fd5c41d8..95bbc36ac145 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
@@ -44,6 +44,9 @@
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
@@ -58,9 +61,6 @@
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java
index b4d4b08dfc20..5b7bff07fc7a 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java
@@ -18,7 +18,6 @@
 package org.apache.beam.runners.flink;
 
 import java.io.IOException;
-
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.metrics.MetricResults;
 import org.joda.time.Duration;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
index 41db61edbb49..c3b979401ff1 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
@@ -18,9 +18,7 @@
 package org.apache.beam.runners.flink.translation.types;
 
 import java.io.IOException;
-
 import org.apache.beam.sdk.coders.Coder;
-
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index 518d6bed308d..e44ad57b7a8f 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -57,6 +57,7 @@
 import org.apache.beam.runners.flink.translation.wrappers.streaming.state.KeyGroupCheckpointedOperator;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.BagState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.join.RawUnionValue;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
@@ -67,7 +68,6 @@
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.BagState;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
index cfe3f9bf66eb..f44e66801e4a 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
@@ -32,20 +32,20 @@
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.MapCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CombineContextFactory;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
index c9b7797ef1a8..d6af4f9be944 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
@@ -35,20 +35,20 @@
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.state.KeyGroupsList;
 import org.apache.flink.runtime.state.KeyedStateBackend;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java
index 3d38f88c193a..bb2a9fff45d9 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkSplitStateInternals.java
@@ -24,19 +24,19 @@
 import org.apache.beam.runners.core.StateTag;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.state.ListStateDescriptor;
 import org.apache.flink.runtime.state.OperatorStateBackend;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
index cea6e0f719fb..9cb742ee7e30 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
@@ -29,22 +29,22 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.InstantCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.sdk.util.CombineContextFactory;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeutils.base.StringSerializer;
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
index bda30e4b8be7..2a51be619c5c 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
@@ -39,6 +39,9 @@
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.PCollectionViewTesting;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.join.RawUnionValue;
@@ -52,9 +55,6 @@
 import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java
index eb2c05f68703..2b96d91e917a 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkBroadcastStateInternalsTest.java
@@ -31,12 +31,12 @@
 import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkBroadcastStateInternals;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
 import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java
index 0e0267bf9cac..40123737d2ae 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkKeyGroupStateInternalsTest.java
@@ -34,9 +34,9 @@
 import org.apache.beam.runners.core.StateTags;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkKeyGroupStateInternals;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.ReadableState;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.ReadableState;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.java.typeutils.GenericTypeInfo;
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java
index 8033a9daf47e..17cd3f5d2c84 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkSplitStateInternalsTest.java
@@ -27,8 +27,8 @@
 import org.apache.beam.runners.core.StateTags;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkSplitStateInternals;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.ReadableState;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.ReadableState;
 import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
 import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
index cd00d9edf234..35d2b786b82d 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
@@ -32,17 +32,17 @@
 import org.apache.beam.runners.flink.translation.wrappers.streaming.state.FlinkStateInternals;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.GroupingState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.java.typeutils.GenericTypeInfo;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java
index 38129ab08216..d2ab3579f75a 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java
@@ -41,14 +41,14 @@
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 70f00fb0f2a0..986ed5c757a1 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -85,6 +85,9 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -97,9 +100,6 @@
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
index afaba3a571f0..5ea0b0331f3c 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
@@ -30,20 +30,20 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateContext;
+import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.CombineFnUtil;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.ReadableState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateContext;
-import org.apache.beam.sdk.util.state.StateContexts;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.joda.time.Instant;
 
 
diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
index 28bbc3cca40a..1db0e86d6afc 100644
--- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
@@ -373,7 +373,7 @@
     <!--[BEAM-420] Non-transient non-serializable instance field in serializable class-->
   </Match>
   <Match>
-    <Class name="org.apache.beam.sdk.util.state.StateSpecs$CombiningStateSpec"/>
+    <Class name="StateSpecs$CombiningStateSpec"/>
     <Method name="equals"/>
     <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS"/>
     <!--[BEAM-421] Class doesn't override equals in superclass-->
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/BagState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
similarity index 96%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/BagState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
index e0eebe598dff..189d1517a302 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/BagState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 /**
  * State containing a bag values. Items can be added to the bag and the contents read out.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/CombiningState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/CombiningState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
index 80e4dc9478f3..6080127d3800 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/CombiningState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/GroupingState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/GroupingState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
index bd7a8d9cf9d6..3a12e79ac4a3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/GroupingState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/MapState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/MapState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
index fb7e807804d0..9f0eee9a59cf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/MapState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import java.util.Map;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ReadableState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ReadableState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
index c3e9936b9305..b29ab266a1ca 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ReadableState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ReadableStates.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ReadableStates.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
index 819eda694a0c..d8df04e8e255 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ReadableStates.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/SetState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/SetState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
index 56ea510411be..14aa6406f4fc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/SetState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 /**
  * State containing no duplicate elements.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/State.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
similarity index 96%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/State.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
index 3a49f017bec7..6b10c9100492 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/State.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 /**
  * Base interface for all state locations.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateBinder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateBinder.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
index 48fa7428dd28..ee4aa78983b6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateBinder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.Combine;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateContext.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
index 887a5f11db1a..110a51569065 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateContexts.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateContexts.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
index 2ce959430569..63afe4f2aada 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateContexts.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateSpec.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
index 8eda218193c8..3b0b8407ca97 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import java.io.Serializable;
 import org.apache.beam.sdk.annotations.Experimental;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
similarity index 99%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateSpecs.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
index 49d57228eb39..09cc4e733bf4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import static com.google.common.base.Preconditions.checkArgument;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ValueState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ValueState.java
similarity index 96%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ValueState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/ValueState.java
index b4322039a6c6..ca97db2fae4b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/ValueState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ValueState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/WatermarkHoldState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/WatermarkHoldState.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
index ae9b700f1fa2..9f6c20383747 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/WatermarkHoldState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
similarity index 95%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/package-info.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
index b9bec1600d01..de5eeeb22468 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
@@ -19,4 +19,4 @@
 /**
  * Defines internal utilities for interacting with pipeline state.
  */
-package org.apache.beam.sdk.util.state;
+package org.apache.beam.sdk.state;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesMapState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesMapState.java
index 9bced418d225..585d8b7e481a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesMapState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesMapState.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.sdk.testing;
 
-import org.apache.beam.sdk.util.state.MapState;
+import org.apache.beam.sdk.state.MapState;
 
 /**
  * Category tag for validation tests which utilize {@link MapState}.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesSetState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesSetState.java
index 6fd74bdf4f13..7d82d227a3c9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesSetState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesSetState.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.sdk.testing;
 
-import org.apache.beam.sdk.util.state.SetState;
+import org.apache.beam.sdk.state.SetState;
 
 /**
  * Category tag for validation tests which utilize {@link SetState}.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index f3d178e16f84..c85893653093 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -27,6 +27,8 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
 import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
@@ -37,8 +39,6 @@
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateSpec;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index ca7427c9bc64..ead256948c86 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -320,7 +320,7 @@ public RestrictionTracker<?> restrictionTracker() {
             }
 
             @Override
-            public org.apache.beam.sdk.util.state.State state(String stateId) {
+            public org.apache.beam.sdk.state.State state(String stateId) {
               throw new UnsupportedOperationException("DoFnTester doesn't support state yet");
             }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
index e1321151ada0..682897965a49 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
@@ -23,16 +23,16 @@
 import com.google.common.collect.Iterables;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.util.TimerSpecs;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.joda.time.Duration;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index 7dd2cdd53ce8..c45311a7b0ab 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -33,6 +33,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
@@ -46,7 +47,6 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.state.StateSpec;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
index 3c44afe7b98f..d5a1a9473d10 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
@@ -19,6 +19,7 @@
 
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundle;
 import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
@@ -28,7 +29,6 @@
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.state.State;
 
 /**
  * Interface for invoking the {@code DoFn} processing methods.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index 3219f9680a55..72ad4b0ef1ce 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -27,6 +27,8 @@
 import java.util.Map;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.StateId;
 import org.apache.beam.sdk.transforms.DoFn.TimerId;
@@ -38,8 +40,6 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateSpec;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
index bac3befc36df..3dfca8c456ae 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -42,6 +42,8 @@
 import java.util.Map;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.StateId;
 import org.apache.beam.sdk.transforms.DoFn.TimerId;
@@ -58,8 +60,6 @@
 import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
-import org.apache.beam.sdk.util.state.State;
-import org.apache.beam.sdk.util.state.StateSpec;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.sdk.values.TypeParameter;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java
index 31d1f645fd7f..f93cb0b50200 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java
@@ -18,9 +18,9 @@
 package org.apache.beam.sdk.util;
 
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.StateContext;
 import org.apache.beam.sdk.transforms.CombineWithContext.Context;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.state.StateContext;
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineFnUtil.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineFnUtil.java
index a3941803c2f0..1b1c35285877 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineFnUtil.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineFnUtil.java
@@ -23,12 +23,12 @@
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.state.StateContext;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.CombineWithContext.Context;
 import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.util.state.StateContext;
 
 /**
  * Static utility methods that create combine function instances.
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 56051a69689d..1d4192381b23 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -64,6 +64,13 @@
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.MapState;
+import org.apache.beam.sdk.state.SetState;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -92,13 +99,6 @@
 import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CombiningState;
-import org.apache.beam.sdk.util.state.MapState;
-import org.apache.beam.sdk.util.state.SetState;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index fe96e87951a4..13e46d517ec1 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -38,6 +38,9 @@
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker.FakeArgumentProvider;
 import org.apache.beam.sdk.transforms.reflect.testhelper.DoFnInvokersTestHelper;
@@ -50,9 +53,6 @@
 import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.UserCodeException;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
 import org.joda.time.Instant;
 import org.junit.Before;
 import org.junit.Rule;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
index f099d5d1b846..27e0b89626be 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -29,6 +29,10 @@
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.ValueState;
+import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.ProcessContextParameter;
@@ -41,10 +45,6 @@
 import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.util.TimerSpecs;
-import org.apache.beam.sdk.util.state.StateSpec;
-import org.apache.beam.sdk.util.state.StateSpecs;
-import org.apache.beam.sdk.util.state.ValueState;
-import org.apache.beam.sdk.util.state.WatermarkHoldState;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.hamcrest.Matcher;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java
index 798e8dca4663..d16671b81b68 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java
@@ -27,10 +27,10 @@
 import java.io.NotSerializableException;
 import java.io.ObjectOutputStream;
 import java.util.List;
+import org.apache.beam.sdk.state.StateContexts;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.CombineWithContext.Context;
 import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.util.state.StateContexts;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;

From ac01ec7afb43ae0bc4198234d1de16a830f95b10 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 10:41:01 -0700
Subject: [PATCH 051/387] Add @Internal and @Experimental to state package

---
 .../org/apache/beam/sdk/state/BagState.java   |  4 +
 .../apache/beam/sdk/state/CombiningState.java |  3 +
 .../apache/beam/sdk/state/GroupingState.java  |  3 +
 .../org/apache/beam/sdk/state/MapState.java   |  3 +
 .../apache/beam/sdk/state/ReadableStates.java |  7 +-
 .../org/apache/beam/sdk/state/SetState.java   |  4 +
 .../java/org/apache/beam/sdk/state/State.java |  4 +
 .../apache/beam/sdk/state/StateBinder.java    |  7 +-
 .../apache/beam/sdk/state/StateContext.java   |  5 +-
 .../apache/beam/sdk/state/StateContexts.java  |  5 +-
 .../org/apache/beam/sdk/state/StateSpec.java  | 18 +++-
 .../org/apache/beam/sdk/state/StateSpecs.java | 88 +++++++++++--------
 .../beam/sdk/state/WatermarkHoldState.java    | 11 ++-
 .../apache/beam/sdk/state/package-info.java   |  2 +-
 14 files changed, 109 insertions(+), 55 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
index 189d1517a302..10fba05d1fbb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
@@ -17,11 +17,15 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+
 /**
  * State containing a bag values. Items can be added to the bag and the contents read out.
  *
  * @param <T> The type of elements in the bag.
  */
+@Experimental(Kind.STATE)
 public interface BagState<T> extends GroupingState<T, Iterable<T>> {
   @Override
   BagState<T> readLater();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
index 6080127d3800..ddda25519e3f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
@@ -17,6 +17,8 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 
 /**
@@ -27,6 +29,7 @@
  * @param <AccumT> the type of accumulator
  * @param <OutputT> the type of value extracted from the state
  */
+@Experimental(Kind.STATE)
 public interface CombiningState<InputT, AccumT, OutputT>
     extends GroupingState<InputT, OutputT> {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
index 3a12e79ac4a3..d99ff25515cf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
@@ -17,6 +17,8 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 
 /**
@@ -26,6 +28,7 @@
  * @param <InputT> the type of values added to the state
  * @param <OutputT> the type of value extracted from the state
  */
+@Experimental(Kind.STATE)
 public interface GroupingState<InputT, OutputT> extends ReadableState<OutputT>, State {
   /**
    * Add a value to the buffer.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
index 9f0eee9a59cf..649c3c7653d8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
@@ -18,6 +18,8 @@
 package org.apache.beam.sdk.state;
 
 import java.util.Map;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
  * An object that maps keys to values.
@@ -27,6 +29,7 @@
  * @param <K> the type of keys maintained by this map
  * @param <V> the type of mapped values
  */
+@Experimental(Kind.STATE)
 public interface MapState<K, V> extends State {
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
index d8df04e8e255..6977a9718bb3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableStates.java
@@ -17,13 +17,12 @@
  */
 package org.apache.beam.sdk.state;
 
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 
 /**
- * Utilities for constructing and manipulating {@link ReadableState} instances.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  */
-@Experimental(Kind.STATE)
+@Internal
 public class ReadableStates {
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
index 14aa6406f4fc..cb9a0e6dd780 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
@@ -17,12 +17,16 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+
 /**
  * State containing no duplicate elements.
  * Items can be added to the set and the contents read out.
  *
  * @param <T> The type of elements in the set.
  */
+@Experimental(Kind.STATE)
 public interface SetState<T> extends GroupingState<T, Iterable<T>> {
   /**
    * Returns true if this set contains the specified element.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
index 6b10c9100492..0c0ca32f2ecc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
@@ -17,12 +17,16 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+
 /**
  * Base interface for all state locations.
  *
  * <p>Specific types of state add appropriate accessors for reading and writing values, see
  * {@link ValueState}, {@link BagState}, and {@link GroupingState}.
  */
+@Experimental(Kind.STATE)
 public interface State {
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
index ee4aa78983b6..af834c3cdc91 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
@@ -17,14 +17,19 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 
 /**
- * Visitor for binding a {@link StateSpec} and to the associated {@link State}.
+ * For internal use only; no backwards-compatibility guarantees.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>Visitor for binding a {@link StateSpec} and to the associated {@link State}.
  */
+@Internal
 public interface StateBinder {
   <T> ValueState<T> bindValue(
       String id, StateSpec<ValueState<T>> spec, Coder<T> coder);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
index 110a51569065..52177cc0a41c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
@@ -17,13 +17,16 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
- * Information accessible the state API.
+ * For internal use only; no backwards-compatibility guarantees.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  */
+@Internal
 public interface StateContext<W extends BoundedWindow> {
   /**
    * Returns the {@code PipelineOptions} specified with the
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
index 63afe4f2aada..2e21a27c1a6a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
@@ -17,13 +17,16 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
- * Factory that produces {@link StateContext} based on different inputs.
+ * For internal use only; no backwards-compatibility guarantees.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  */
+@Internal
 public class StateContexts {
   private static final StateContext<BoundedWindow> NULL_CONTEXT =
       new StateContext<BoundedWindow>() {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
index 3b0b8407ca97..b0412bf49c6d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
@@ -20,6 +20,7 @@
 import java.io.Serializable;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.Coder;
 
 /**
@@ -32,22 +33,31 @@
 public interface StateSpec<StateT extends State> extends Serializable {
 
   /**
-   * Use the {@code binder} to create an instance of {@code StateT} appropriate for this address.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Use the {@code binder} to create an instance of {@code StateT} appropriate for this address.
    */
+  @Internal
   StateT bind(String id, StateBinder binder);
 
   /**
-   * Given {code coders} are inferred from type arguments defined for this class. Coders which are
-   * already set should take precedence over offered coders.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Given {code coders} are inferred from type arguments defined for this class. Coders which
+   * are already set should take precedence over offered coders.
    *
    * @param coders Array of coders indexed by the type arguments order. Entries might be null if the
    *     coder could not be inferred.
    */
+  @Internal
   void offerCoders(Coder[] coders);
 
   /**
-   * Validates that this {@link StateSpec} has been specified correctly and finalizes it.
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Validates that this {@link StateSpec} has been specified correctly and finalizes it.
    * Automatically invoked when the pipeline is built.
    */
+  @Internal
   void finishSpecifying();
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
index 09cc4e733bf4..8a3c87e4c6a4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
@@ -23,6 +23,7 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
@@ -94,42 +95,6 @@ StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
     return combiningInternal(accumCoder, combineFn);
   }
 
-  /**
-   * Create a state spec for values that use a {@link CombineFn} to automatically merge multiple
-   * {@code InputT}s into a single {@code OutputT}.
-   *
-   * <p>This determines the {@code Coder<AccumT>} from the given {@code Coder<InputT>}, and should
-   * only be used to initialize static values.
-   */
-  public static <InputT, AccumT, OutputT>
-  StateSpec<CombiningState<InputT, AccumT, OutputT>>
-  combiningFromInputInternal(
-              Coder<InputT> inputCoder, CombineFn<InputT, AccumT, OutputT> combineFn) {
-    try {
-      Coder<AccumT> accumCoder = combineFn.getAccumulatorCoder(STANDARD_REGISTRY, inputCoder);
-      return combiningInternal(accumCoder, combineFn);
-    } catch (CannotProvideCoderException e) {
-      throw new IllegalArgumentException(
-          "Unable to determine accumulator coder for "
-              + combineFn.getClass().getSimpleName()
-              + " from "
-              + inputCoder,
-          e);
-    }
-  }
-
-  private static <InputT, AccumT, OutputT>
-  StateSpec<CombiningState<InputT, AccumT, OutputT>> combiningInternal(
-          Coder<AccumT> accumCoder, CombineFn<InputT, AccumT, OutputT> combineFn) {
-    return new CombiningStateSpec<InputT, AccumT, OutputT>(accumCoder, combineFn);
-  }
-
-  private static <InputT, AccumT, OutputT>
-  StateSpec<CombiningState<InputT, AccumT, OutputT>> combiningInternal(
-      Coder<AccumT> accumCoder, CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
-    return new CombiningWithContextStateSpec<InputT, AccumT, OutputT>(accumCoder, combineFn);
-  }
-
   /**
    * Create a state spec that is optimized for adding values frequently, and occasionally retrieving
    * all the values that have been added.
@@ -172,13 +137,62 @@ public static <K, V> StateSpec<MapState<K, V>> map(Coder<K> keyCoder, Coder<V> v
     return new MapStateSpec<>(keyCoder, valueCoder);
   }
 
-  /** Create a state spec for holding the watermark. */
+  /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Create a state spec for values that use a {@link CombineFn} to automatically merge multiple
+   * {@code InputT}s into a single {@code OutputT}.
+   *
+   * <p>This determines the {@code Coder<AccumT>} from the given {@code Coder<InputT>}, and should
+   * only be used to initialize static values.
+   */
+  @Internal
+  public static <InputT, AccumT, OutputT>
+      StateSpec<CombiningState<InputT, AccumT, OutputT>> combiningFromInputInternal(
+          Coder<InputT> inputCoder, CombineFn<InputT, AccumT, OutputT> combineFn) {
+    try {
+      Coder<AccumT> accumCoder = combineFn.getAccumulatorCoder(STANDARD_REGISTRY, inputCoder);
+      return combiningInternal(accumCoder, combineFn);
+    } catch (CannotProvideCoderException e) {
+      throw new IllegalArgumentException(
+          "Unable to determine accumulator coder for "
+              + combineFn.getClass().getSimpleName()
+              + " from "
+              + inputCoder,
+          e);
+    }
+  }
+
+  private static <InputT, AccumT, OutputT>
+  StateSpec<CombiningState<InputT, AccumT, OutputT>> combiningInternal(
+          Coder<AccumT> accumCoder, CombineFn<InputT, AccumT, OutputT> combineFn) {
+    return new CombiningStateSpec<InputT, AccumT, OutputT>(accumCoder, combineFn);
+  }
+
+  private static <InputT, AccumT, OutputT>
+  StateSpec<CombiningState<InputT, AccumT, OutputT>> combiningInternal(
+      Coder<AccumT> accumCoder, CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
+    return new CombiningWithContextStateSpec<InputT, AccumT, OutputT>(accumCoder, combineFn);
+  }
+
+  /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Create a state spec for a watermark hold.
+   */
+  @Internal
   public static
       StateSpec<WatermarkHoldState> watermarkStateInternal(
           TimestampCombiner timestampCombiner) {
     return new WatermarkStateSpecInternal(timestampCombiner);
   }
 
+  /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Convert a combining state spec to a bag of accumulators.
+   */
+  @Internal
   public static <InputT, AccumT, OutputT>
       StateSpec<BagState<AccumT>> convertToBagSpecInternal(
           StateSpec<CombiningState<InputT, AccumT, OutputT>> combiningSpec) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
index 9f6c20383747..38e2cbc27b64 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
@@ -17,18 +17,17 @@
  */
 package org.apache.beam.sdk.state;
 
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.joda.time.Instant;
 
 /**
- * A {@link State} accepting and aggregating output timestamps, which determines the time to which
- * the output watermark must be held.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  *
- * <p><b><i>For internal use only. This API may change at any time.</i></b>
+ * <p>A {@link State} accepting and aggregating output timestamps, which determines the time to
+ * which the output watermark must be held.
  */
-@Experimental(Kind.STATE)
+@Internal
 public interface WatermarkHoldState extends GroupingState<Instant, Instant> {
   /**
    * Return the {@link TimestampCombiner} which will be used to determine a watermark hold time
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
index de5eeeb22468..d8b8e9266c2e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/package-info.java
@@ -17,6 +17,6 @@
  */
 
 /**
- * Defines internal utilities for interacting with pipeline state.
+ * Classes and interfaces for interacting with state.
  */
 package org.apache.beam.sdk.state;

From ca41af8fe4711ab4a81c2a33746a64e64fb0ca37 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 12:46:46 -0700
Subject: [PATCH 052/387] Move user-facing timer-related classes out of util

---
 .../translation/operators/ApexGroupByKeyOperator.java     | 2 +-
 .../apex/translation/operators/ApexTimerInternals.java    | 2 +-
 .../translation/operators/ApexTimerInternalsTest.java     | 2 +-
 .../apache/beam/runners/core/construction/Triggers.java   | 2 +-
 .../runners/core/construction/PTransformMatchersTest.java | 8 ++++----
 .../java/org/apache/beam/runners/core/DoFnAdapters.java   | 2 +-
 .../java/org/apache/beam/runners/core/DoFnRunner.java     | 2 +-
 .../apache/beam/runners/core/InMemoryTimerInternals.java  | 2 +-
 .../beam/runners/core/LateDataDroppingDoFnRunner.java     | 2 +-
 ...tputAndTimeBoundedSplittableProcessElementInvoker.java | 2 +-
 .../org/apache/beam/runners/core/ProcessFnRunner.java     | 2 +-
 .../beam/runners/core/PushbackSideInputDoFnRunner.java    | 2 +-
 .../main/java/org/apache/beam/runners/core/ReduceFn.java  | 2 +-
 .../apache/beam/runners/core/ReduceFnContextFactory.java  | 4 ++--
 .../java/org/apache/beam/runners/core/ReduceFnRunner.java | 2 +-
 .../org/apache/beam/runners/core/SimpleDoFnRunner.java    | 6 +++---
 .../org/apache/beam/runners/core/SimpleOldDoFnRunner.java | 2 +-
 .../runners/core/SimplePushbackSideInputDoFnRunner.java   | 2 +-
 .../org/apache/beam/runners/core/SplittableParDo.java     | 2 +-
 .../org/apache/beam/runners/core/StatefulDoFnRunner.java  | 2 +-
 .../java/org/apache/beam/runners/core/TimerInternals.java | 2 +-
 .../triggers/AfterDelayFromFirstElementStateMachine.java  | 2 +-
 .../core/triggers/AfterProcessingTimeStateMachine.java    | 2 +-
 .../AfterSynchronizedProcessingTimeStateMachine.java      | 2 +-
 .../runners/core/triggers/AfterWatermarkStateMachine.java | 2 +-
 .../runners/core/triggers/DefaultTriggerStateMachine.java | 2 +-
 .../beam/runners/core/triggers/TriggerStateMachine.java   | 2 +-
 .../core/triggers/TriggerStateMachineContextFactory.java  | 4 ++--
 .../runners/core/triggers/TriggerStateMachineRunner.java  | 2 +-
 .../beam/runners/core/InMemoryTimerInternalsTest.java     | 2 +-
 .../apache/beam/runners/core/KeyedWorkItemCoderTest.java  | 2 +-
 .../org/apache/beam/runners/core/ReduceFnRunnerTest.java  | 2 +-
 .../java/org/apache/beam/runners/core/ReduceFnTester.java | 2 +-
 .../apache/beam/runners/core/SimpleDoFnRunnerTest.java    | 8 ++++----
 .../core/SimplePushbackSideInputDoFnRunnerTest.java       | 2 +-
 .../org/apache/beam/runners/core/TimerInternalsTest.java  | 2 +-
 .../runners/core/triggers/TriggerStateMachineTester.java  | 4 ++--
 .../runners/core/triggers/TriggerStateMachinesTest.java   | 2 +-
 .../apache/beam/runners/direct/DirectTimerInternals.java  | 2 +-
 .../org/apache/beam/runners/direct/WatermarkManager.java  | 2 +-
 .../beam/runners/direct/DirectTimerInternalsTest.java     | 2 +-
 ...oFnLifecycleManagerRemovingTransformEvaluatorTest.java | 2 +-
 .../apache/beam/runners/direct/EvaluationContextTest.java | 2 +-
 .../apache/beam/runners/direct/WatermarkManagerTest.java  | 2 +-
 .../flink/metrics/DoFnRunnerWithMetricsUpdate.java        | 2 +-
 .../translation/wrappers/streaming/DoFnOperator.java      | 2 +-
 .../beam/runners/flink/streaming/DoFnOperatorTest.java    | 8 ++++----
 .../beam/runners/spark/stateful/SparkTimerInternals.java  | 2 +-
 .../runners/spark/translation/DoFnRunnerWithMetrics.java  | 2 +-
 .../java/org/apache/beam/sdk/options/ValueProviders.java  | 1 -
 .../org/apache/beam/sdk/{util => state}/TimeDomain.java   | 2 +-
 .../java/org/apache/beam/sdk/{util => state}/Timer.java   | 2 +-
 .../org/apache/beam/sdk/{util => state}/TimerSpec.java    | 2 +-
 .../org/apache/beam/sdk/{util => state}/TimerSpecs.java   | 4 ++--
 .../java/org/apache/beam/sdk/{util => state}/Timers.java  | 2 +-
 .../org/apache/beam/sdk/testing/TestPipelineOptions.java  | 2 --
 .../main/java/org/apache/beam/sdk/transforms/DoFn.java    | 6 +++---
 .../java/org/apache/beam/sdk/transforms/DoFnTester.java   | 2 +-
 .../org/apache/beam/sdk/transforms/GroupIntoBatches.java  | 8 ++++----
 .../main/java/org/apache/beam/sdk/transforms/Regex.java   | 1 -
 .../transforms/reflect/ByteBuddyDoFnInvokerFactory.java   | 2 +-
 .../apache/beam/sdk/transforms/reflect/DoFnInvoker.java   | 2 +-
 .../apache/beam/sdk/transforms/reflect/DoFnSignature.java | 4 ++--
 .../beam/sdk/transforms/reflect/DoFnSignatures.java       | 4 ++--
 .../beam/sdk/transforms/windowing/AfterWatermark.java     | 2 +-
 .../src/main/java/org/apache/beam/sdk/util/ClassPath.java | 1 +
 .../beam/sdk/io/DrunkWritableByteChannelFactory.java      | 1 -
 .../test/java/org/apache/beam/sdk/io/TFRecordIOTest.java  | 2 --
 .../java/org/apache/beam/sdk/metrics/GaugeCellTest.java   | 1 +
 .../org/apache/beam/sdk/options/PipelineOptionsTest.java  | 1 -
 .../java/org/apache/beam/sdk/transforms/ParDoTest.java    | 8 ++++----
 .../java/org/apache/beam/sdk/transforms/RegexTest.java    | 2 --
 .../beam/sdk/transforms/reflect/DoFnInvokersTest.java     | 8 ++++----
 .../beam/sdk/transforms/reflect/DoFnSignaturesTest.java   | 8 ++++----
 .../beam/sdk/transforms/reflect/OnTimerInvokersTest.java  | 6 +++---
 .../reflect/testhelper/DoFnInvokersTestHelper.java        | 6 +++---
 76 files changed, 104 insertions(+), 112 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => state}/TimeDomain.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => state}/Timer.java (98%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => state}/TimerSpec.java (97%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => state}/TimerSpecs.java (91%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => state}/Timers.java (98%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
index 4fdb60017c8e..6131bdf41ae9 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
@@ -45,11 +45,11 @@
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.NullSideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java
index 15ccbee1d737..1eb224c266a7 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java
@@ -35,8 +35,8 @@
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java
index ee142e2942e8..7b52223e5478 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java
@@ -32,8 +32,8 @@
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.runners.core.TimerInternals.TimerDataCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 import org.junit.Test;
 
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
index 81f738da0dd5..5e735715b3ee 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.AfterAll;
 import org.apache.beam.sdk.transforms.windowing.AfterEach;
 import org.apache.beam.sdk.transforms.windowing.AfterFirst;
@@ -43,7 +44,6 @@
 import org.apache.beam.sdk.transforms.windowing.Trigger;
 import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger;
 import org.apache.beam.sdk.util.ReshuffleTrigger;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index 278c12bf8bcf..ba935a4e9a0b 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -39,6 +39,10 @@
 import org.apache.beam.sdk.runners.PTransformMatcher;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
@@ -57,10 +61,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.PCollectionViews;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
index 849400f1d06d..af59a40b7518 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
@@ -21,6 +21,7 @@
 import org.apache.beam.runners.core.OldDoFn.Context;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.Timer;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
@@ -34,7 +35,6 @@
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.joda.time.Duration;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunner.java
index b29adcc1c7da..30648f6e5820 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunner.java
@@ -17,9 +17,9 @@
  */
 package org.apache.beam.runners.core;
 
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java
index f8aaa0904971..e68bb2407c52 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java
@@ -27,8 +27,8 @@
 import java.util.NavigableSet;
 import java.util.TreeSet;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
index d6ba8f5ca46e..872ee3e06657 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
@@ -23,8 +23,8 @@
 import com.google.common.collect.Iterables;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index 8e80a694d437..16bdfa361c0d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -28,6 +28,7 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.Timer;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
@@ -36,7 +37,6 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
index 3ae3f5068c46..7cbf0d214053 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
@@ -25,10 +25,10 @@
 import java.util.Collections;
 import org.apache.beam.runners.core.StateNamespaces.WindowNamespace;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Instant;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
index bab1dc7317fc..8f21086794d9 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java
@@ -17,9 +17,9 @@
  */
 package org.apache.beam.runners.core;
 
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
index bcc47a84600b..8e2ff7368b1d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
@@ -19,9 +19,9 @@
 
 import java.io.Serializable;
 import org.apache.beam.sdk.state.ReadableState;
+import org.apache.beam.sdk.state.Timers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.Timers;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
index cb1a15964e32..550b9b9769b4 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
@@ -31,11 +31,11 @@
 import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.state.StateContext;
 import org.apache.beam.sdk.state.StateContexts;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timers;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Instant;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
index d3dc067e31ac..a949eddfb116 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
@@ -45,6 +45,7 @@
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -54,7 +55,6 @@
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 200a43b5952f..2567bbaa3b57 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -34,6 +34,9 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
 import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
@@ -51,9 +54,6 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
index 4c3149acba5c..7454dc9248d4 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
@@ -29,6 +29,7 @@
 import org.apache.beam.runners.core.ExecutionContext.StepContext;
 import org.apache.beam.runners.core.OldDoFn.RequiresWindowAccess;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
@@ -36,7 +37,6 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java
index 50d301bc18e6..36a04d9e2f04 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java
@@ -23,9 +23,9 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Instant;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
index 1fda7d92a176..74ffeb635339 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
@@ -29,6 +29,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -46,7 +47,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
index 0a7701aed8d9..346694566084 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
@@ -23,13 +23,13 @@
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
index 375cdf97a257..21fe4301ea57 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
@@ -30,9 +30,9 @@
 import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
index b972985fa3ac..8d8d0de40bbe 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java
@@ -32,10 +32,10 @@
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.state.CombiningState;
 import org.apache.beam.sdk.state.GroupingState;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.Combine.Holder;
 import org.apache.beam.sdk.transforms.Min;
 import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.joda.time.format.PeriodFormat;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterProcessingTimeStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterProcessingTimeStateMachine.java
index eaf5613cc795..e813d33dd963 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterProcessingTimeStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterProcessingTimeStateMachine.java
@@ -21,8 +21,8 @@
 import java.util.Objects;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java
index 07fab223739c..e1cd8978abb6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java
@@ -21,8 +21,8 @@
 import java.util.Collections;
 import java.util.List;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 
 // This should not really have the superclass https://issues.apache.org/jira/browse/BEAM-1486
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java
index 1b117d2e6923..c9eee15b857f 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java
@@ -24,7 +24,7 @@
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.triggers.TriggerStateMachine.OnceTriggerStateMachine;
 import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.state.TimeDomain;
 
 /**
  * {@code AfterWatermark} triggers fire based on progress of the system watermark. This time is a
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/DefaultTriggerStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/DefaultTriggerStateMachine.java
index be4dd681c603..4ab9fc9dcfc6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/DefaultTriggerStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/DefaultTriggerStateMachine.java
@@ -18,7 +18,7 @@
 package org.apache.beam.runners.core.triggers;
 
 import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.state.TimeDomain;
 
 /**
  * A trigger that is equivalent to {@code Repeatedly.forever(AfterWatermark.pastEndOfWindow())}. See
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java
index d622ac91131d..6a2cf0c91a20 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java
@@ -24,10 +24,10 @@
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.MergingStateAccessor;
 import org.apache.beam.runners.core.StateAccessor;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java
index 84b0453d12fc..811f30c6b5a9 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java
@@ -35,10 +35,10 @@
 import org.apache.beam.runners.core.triggers.TriggerStateMachine.TriggerInfo;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timers;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
index 324d44d7d515..88ea6efccb7b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java
@@ -29,9 +29,9 @@
 import org.apache.beam.runners.core.StateTag;
 import org.apache.beam.runners.core.StateTags;
 import org.apache.beam.sdk.coders.BitSetCoder;
+import org.apache.beam.sdk.state.Timers;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.Timers;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryTimerInternalsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryTimerInternalsTest.java
index 022626030204..704942198d3b 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryTimerInternalsTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryTimerInternalsTest.java
@@ -22,7 +22,7 @@
 import static org.junit.Assert.assertThat;
 
 import org.apache.beam.runners.core.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.joda.time.Instant;
 import org.junit.Test;
 import org.junit.runner.RunWith;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java
index 4d5963492758..4057c25b48b5 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java
@@ -21,9 +21,9 @@
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.testing.CoderProperties;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.joda.time.Instant;
 import org.junit.Test;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
index 7b91151dc1d4..2b400a40e9a5 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
@@ -45,6 +45,7 @@
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.CombineWithContext.Context;
 import org.apache.beam.sdk.transforms.Sum;
@@ -70,7 +71,6 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
index 573855fd8d52..eea30e912e02 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
@@ -51,6 +51,7 @@
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -64,7 +65,6 @@
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index b8fc64ea7a55..d793584ca446 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -33,16 +33,16 @@
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.NullSideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java
index ba3f9263f884..dabc4f013b86 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java
@@ -29,6 +29,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Sum;
@@ -39,7 +40,6 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.IdentitySideInputWindowFn;
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/TimerInternalsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/TimerInternalsTest.java
index 768c974d217e..af270d9fd075 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/TimerInternalsTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/TimerInternalsTest.java
@@ -25,10 +25,10 @@
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.runners.core.TimerInternals.TimerDataCoder;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.testing.CoderProperties;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 import org.junit.Test;
 import org.junit.runner.RunWith;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
index 1bdcff25fd2a..51f467ae8171 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
@@ -44,12 +44,12 @@
 import org.apache.beam.runners.core.StateNamespaces.WindowNamespace;
 import org.apache.beam.runners.core.TestInMemoryStateInternals;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timers;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
index 7ff3478254a7..5158f505101d 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java
@@ -23,7 +23,7 @@
 import static org.junit.Assert.assertThat;
 
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.joda.time.Duration;
 import org.junit.Test;
 import org.junit.runner.RunWith;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
index 67d568da80dd..a099368855ff 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
@@ -23,7 +23,7 @@
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate.TimerUpdateBuilder;
 import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
-import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index b576e00b0beb..21ba734c017c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -55,10 +55,10 @@
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java
index c8ac695cb091..f80515fba9f7 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java
@@ -28,7 +28,7 @@
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate.TimerUpdateBuilder;
 import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
 import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerRemovingTransformEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerRemovingTransformEvaluatorTest.java
index 1ac4d6d2773a..d98e51d83a50 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerRemovingTransformEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerRemovingTransformEvaluatorTest.java
@@ -29,10 +29,10 @@
 
 import org.apache.beam.runners.core.StateNamespaces;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 3eff11a3436a..63864f0856e2 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -44,6 +44,7 @@
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
@@ -54,7 +55,6 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
index e6506a812221..e1e6ab59cdd4 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
@@ -43,6 +43,7 @@
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
@@ -55,7 +56,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
index 29a1a5248682..dae91fe007fe 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
@@ -21,8 +21,8 @@
 import java.io.IOException;
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.joda.time.Instant;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index e44ad57b7a8f..5287b85d0bd6 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -58,6 +58,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.join.RawUnionValue;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
@@ -65,7 +66,6 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
index 2a51be619c5c..80dfa07288c4 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
@@ -41,6 +41,10 @@
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.PCollectionViewTesting;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -49,10 +53,6 @@
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
index 646e2699e9fb..107915f7f43d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
@@ -32,8 +32,8 @@
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.spark.broadcast.Broadcast;
 import org.joda.time.Instant;
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
index fa9a9c2599f0..d74b25373387 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
@@ -24,8 +24,8 @@
 import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.spark.Accumulator;
 import org.joda.time.Instant;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
index e2355eeb98fa..1cc46fe4a83f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java
@@ -21,7 +21,6 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-
 import java.io.IOException;
 import java.util.Map;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimeDomain.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimeDomain.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
index b067ab8cda85..e8149158965d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimeDomain.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.state;
 
 /**
  * {@code TimeDomain} specifies whether an operation is based on
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timer.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
index 9727969fcc28..9458906721b5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.transforms.DoFn;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpec.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerSpec.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpec.java
index 5b7717b1b5bb..f08f43e5a8e1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpec.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.state;
 
 import java.io.Serializable;
 import org.apache.beam.sdk.annotations.Experimental;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
similarity index 91%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerSpecs.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
index f289ccdc4770..9efac69671c7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.state;
 
 import com.google.auto.value.AutoValue;
 import org.apache.beam.sdk.annotations.Experimental;
@@ -28,7 +28,7 @@
 public class TimerSpecs {
 
   public static TimerSpec timer(TimeDomain timeDomain) {
-    return new AutoValue_TimerSpecs_SimpleTimerSpec(timeDomain);
+    return new org.apache.beam.sdk.util.AutoValue_TimerSpecs_SimpleTimerSpec(timeDomain);
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timers.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java
index 7415d7a2f99e..dcdaf00f7eb5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Timers.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.state;
 
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipelineOptions.java
index e1ad47bfceec..206bc1f343c4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipelineOptions.java
@@ -18,9 +18,7 @@
 package org.apache.beam.sdk.testing;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index c85893653093..befba1dd71d7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -29,6 +29,9 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
 import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
@@ -36,9 +39,6 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index ead256948c86..8a03f3c0064a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -33,6 +33,7 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.state.Timer;
 import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
@@ -43,7 +44,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
index 682897965a49..b02336339ffa 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
@@ -27,12 +27,12 @@
 import org.apache.beam.sdk.state.CombiningState;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.joda.time.Duration;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
index 690d321c77b6..8913e749fe05 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
@@ -21,7 +21,6 @@
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
index 2b6d5be1c595..8ae2c65a34ec 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
@@ -63,6 +63,7 @@
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.state.Timer;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.OnTimerMethod;
@@ -77,7 +78,6 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
 import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.util.Timer;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
index d5a1a9473d10..6fd40523c28c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
@@ -20,6 +20,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.state.State;
+import org.apache.beam.sdk.state.Timer;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.FinishBundle;
 import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
@@ -28,7 +29,6 @@
 import org.apache.beam.sdk.transforms.DoFn.TimerId;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.Timer;
 
 /**
  * Interface for invoking the {@code DoFn} processing methods.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index 72ad4b0ef1ce..0b4bf90071a8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -29,6 +29,8 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.StateId;
 import org.apache.beam.sdk.transforms.DoFn.TimerId;
@@ -38,8 +40,6 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
index 3dfca8c456ae..bb191b124fdf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -44,6 +44,8 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.StateId;
 import org.apache.beam.sdk.transforms.DoFn.TimerId;
@@ -57,8 +59,6 @@
 import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java
index 14a8c98a7787..5648895f1c5b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java
@@ -24,8 +24,8 @@
 import java.util.Objects;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger;
-import org.apache.beam.sdk.util.TimeDomain;
 import org.joda.time.Instant;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
index 35632ed063bf..271bce0fed58 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
@@ -19,6 +19,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.common.annotations.Beta;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.CharMatcher;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java
index 79f09961db2a..6615a2e9d333 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DrunkWritableByteChannelFactory.java
@@ -20,7 +20,6 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.WritableByteChannel;
-
 import org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory;
 import org.apache.beam.sdk.util.MimeTypes;
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java
index 9a9e840801c7..d564d3b2d847 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java
@@ -30,7 +30,6 @@
 import com.google.common.collect.Lists;
 import com.google.common.io.BaseEncoding;
 import com.google.common.io.ByteStreams;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -43,7 +42,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.PAssert;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/GaugeCellTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/GaugeCellTest.java
index d8ef92825be7..7dbfc4ba822b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/GaugeCellTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/GaugeCellTest.java
@@ -19,6 +19,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertThat;
+
 import org.junit.Test;
 
 /**
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java
index a13aa011b2ff..5e3211fc1dec 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java
@@ -23,7 +23,6 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableMap;
-
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 1d4192381b23..ffdf3d0506a8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -70,6 +70,10 @@
 import org.apache.beam.sdk.state.SetState;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.PAssert;
@@ -94,10 +98,6 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
index ceebccad9887..969c4ea25c9c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
@@ -20,13 +20,11 @@
 import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
-
 import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
-
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index 13e46d517ec1..19b60925c361 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -40,6 +40,10 @@
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker.FakeArgumentProvider;
@@ -48,10 +52,6 @@
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
index 27e0b89626be..5b97a553c28d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -31,6 +31,10 @@
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -41,10 +45,6 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignaturesTestUtils.FakeDoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.Timer;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.hamcrest.Matcher;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java
index 177f15f39224..d31795216943 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java
@@ -22,11 +22,11 @@
 import static org.junit.Assert.assertThat;
 import static org.mockito.Mockito.when;
 
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/testhelper/DoFnInvokersTestHelper.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/testhelper/DoFnInvokersTestHelper.java
index 95e7c4999542..f9fcc5881335 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/testhelper/DoFnInvokersTestHelper.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/testhelper/DoFnInvokersTestHelper.java
@@ -21,12 +21,12 @@
 import static org.junit.Assert.assertNull;
 import static org.mockito.Mockito.verify;
 
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokersTest;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.TimerSpec;
-import org.apache.beam.sdk.util.TimerSpecs;
 
 /**
  * Test helper for {@link DoFnInvokersTest}, which needs to test package-private access to DoFns in

From b4c808a045d493e93d83c815dc7520c6b213858a Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 12:31:02 -0700
Subject: [PATCH 053/387] Mark TimeDomain experimental alongside Timers;
 improve javadoc

---
 .../org/apache/beam/sdk/state/TimeDomain.java  | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
index e8149158965d..0526453067b8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
@@ -17,27 +17,29 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.annotations.Experimental;
+
 /**
- * {@code TimeDomain} specifies whether an operation is based on
- * timestamps of elements or current "real-world" time as reported while processing.
+ * {@link TimeDomain} specifies whether an operation is based on timestamps of elements or current
+ * "real-world" time as reported while processing.
  */
+@Experimental(Experimental.Kind.TIMERS)
 public enum TimeDomain {
   /**
-   * The {@code EVENT_TIME} domain corresponds to the timestamps on the elements. Time advances
-   * on the system watermark advances.
+   * The {@link #EVENT_TIME} domain corresponds to the timestamps on the elements. Time advances on
+   * the system watermark advances.
    */
   EVENT_TIME,
 
   /**
-   * The {@code PROCESSING_TIME} domain corresponds to the current to the current (system) time.
+   * The {@link #PROCESSING_TIME} domain corresponds to the current to the current (system) time.
    * This is advanced during execution of the pipeline.
    */
   PROCESSING_TIME,
 
   /**
-   * Same as the {@code PROCESSING_TIME} domain, except it won't fire a timer set for time
-   * {@code T} until all timers from earlier stages set for a time earlier than {@code T} have
-   * fired.
+   * Same as the {@link #PROCESSING_TIME} domain, except it won't fire a timer set for time <i>t</i>
+   * until all timers from earlier stages set for a time earlier than <i>t</i> have fired.
    */
   SYNCHRONIZED_PROCESSING_TIME
 }

From b61e5bb2b2e1951c62e7defb752561a0a1af5456 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 2 May 2017 12:53:26 -0700
Subject: [PATCH 054/387] Move WindowingStrategy from util to values

WindowingStrategy is a property on PCollection that transform authors
regularly mess with. It is part of the public API.
---
 .../apex/translation/operators/ApexGroupByKeyOperator.java    | 2 +-
 .../runners/apex/translation/operators/ApexParDoOperator.java | 2 +-
 .../runners/apex/translation/ApexGroupByKeyOperatorTest.java  | 2 +-
 .../beam/runners/apex/translation/ParDoTranslatorTest.java    | 2 +-
 .../apache/beam/runners/core/construction/PCollections.java   | 2 +-
 .../beam/runners/core/construction/PrimitiveCreate.java       | 2 +-
 .../apache/beam/runners/core/construction/SdkComponents.java  | 2 +-
 .../beam/runners/core/construction/WindowingStrategies.java   | 4 ++--
 .../beam/runners/core/construction/PCollectionsTest.java      | 2 +-
 .../runners/core/construction/PTransformMatchersTest.java     | 2 +-
 .../runners/core/construction/ReplacementOutputsTest.java     | 2 +-
 .../beam/runners/core/construction/SdkComponentsTest.java     | 4 ++--
 .../runners/core/construction/WindowingStrategiesTest.java    | 4 ++--
 .../main/java/org/apache/beam/runners/core/DoFnRunners.java   | 2 +-
 .../runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java    | 2 +-
 .../beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java  | 2 +-
 .../runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java    | 2 +-
 .../apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java | 2 +-
 .../apache/beam/runners/core/LateDataDroppingDoFnRunner.java  | 2 +-
 .../main/java/org/apache/beam/runners/core/LateDataUtils.java | 2 +-
 .../main/java/org/apache/beam/runners/core/NonEmptyPanes.java | 4 ++--
 .../src/main/java/org/apache/beam/runners/core/ReduceFn.java  | 2 +-
 .../org/apache/beam/runners/core/ReduceFnContextFactory.java  | 2 +-
 .../java/org/apache/beam/runners/core/ReduceFnRunner.java     | 4 ++--
 .../java/org/apache/beam/runners/core/SimpleDoFnRunner.java   | 2 +-
 .../org/apache/beam/runners/core/SimpleOldDoFnRunner.java     | 2 +-
 .../java/org/apache/beam/runners/core/SplittableParDo.java    | 2 +-
 .../java/org/apache/beam/runners/core/StatefulDoFnRunner.java | 2 +-
 .../main/java/org/apache/beam/runners/core/WatermarkHold.java | 2 +-
 .../core/GroupAlsoByWindowViaOutputBufferDoFnTest.java        | 2 +-
 .../beam/runners/core/GroupAlsoByWindowsProperties.java       | 2 +-
 .../beam/runners/core/LateDataDroppingDoFnRunnerTest.java     | 2 +-
 .../java/org/apache/beam/runners/core/ReduceFnRunnerTest.java | 4 ++--
 .../java/org/apache/beam/runners/core/ReduceFnTester.java     | 4 ++--
 .../org/apache/beam/runners/core/SideInputHandlerTest.java    | 2 +-
 .../org/apache/beam/runners/core/SimpleDoFnRunnerTest.java    | 2 +-
 .../org/apache/beam/runners/core/SplittableParDoTest.java     | 2 +-
 .../org/apache/beam/runners/core/StatefulDoFnRunnerTest.java  | 2 +-
 .../beam/runners/core/triggers/TriggerStateMachineTester.java | 2 +-
 .../java/org/apache/beam/runners/direct/DirectGroupByKey.java | 2 +-
 .../org/apache/beam/runners/direct/EvaluationContext.java     | 2 +-
 .../runners/direct/GroupAlsoByWindowEvaluatorFactory.java     | 2 +-
 .../java/org/apache/beam/runners/direct/ParDoEvaluator.java   | 2 +-
 .../apache/beam/runners/direct/ParDoMultiOverrideFactory.java | 2 +-
 .../org/apache/beam/runners/direct/SideInputContainer.java    | 2 +-
 .../direct/SplittableProcessElementsEvaluatorFactory.java     | 2 +-
 .../beam/runners/direct/StatefulParDoEvaluatorFactory.java    | 2 +-
 .../beam/runners/direct/TestStreamEvaluatorFactory.java       | 2 +-
 .../apache/beam/runners/direct/WatermarkCallbackExecutor.java | 2 +-
 .../org/apache/beam/runners/direct/CommittedResultTest.java   | 2 +-
 .../org/apache/beam/runners/direct/EvaluationContextTest.java | 2 +-
 .../beam/runners/direct/KeyedPValueTrackingVisitorTest.java   | 2 +-
 .../apache/beam/runners/direct/SideInputContainerTest.java    | 2 +-
 .../runners/direct/StatefulParDoEvaluatorFactoryTest.java     | 2 +-
 .../apache/beam/runners/direct/ViewOverrideFactoryTest.java   | 2 +-
 .../beam/runners/direct/WatermarkCallbackExecutorTest.java    | 2 +-
 .../beam/runners/direct/WriteWithShardingFactoryTest.java     | 2 +-
 .../beam/runners/flink/FlinkBatchTransformTranslators.java    | 2 +-
 .../beam/runners/flink/FlinkBatchTranslationContext.java      | 2 +-
 .../runners/flink/FlinkStreamingTransformTranslators.java     | 2 +-
 .../translation/functions/AbstractFlinkCombineRunner.java     | 2 +-
 .../flink/translation/functions/FlinkDoFnFunction.java        | 2 +-
 .../functions/FlinkMergingNonShuffleReduceFunction.java       | 2 +-
 .../translation/functions/FlinkPartialReduceFunction.java     | 2 +-
 .../flink/translation/functions/FlinkReduceFunction.java      | 2 +-
 .../flink/translation/functions/FlinkSideInputReader.java     | 2 +-
 .../translation/functions/FlinkStatefulDoFnFunction.java      | 2 +-
 .../translation/functions/HashingFlinkCombineRunner.java      | 2 +-
 .../translation/functions/SortingFlinkCombineRunner.java      | 2 +-
 .../flink/translation/wrappers/streaming/DoFnOperator.java    | 2 +-
 .../wrappers/streaming/SplittableDoFnOperator.java            | 2 +-
 .../translation/wrappers/streaming/WindowDoFnOperator.java    | 2 +-
 .../org/apache/beam/runners/flink/PipelineOptionsTest.java    | 2 +-
 .../apache/beam/runners/flink/streaming/DoFnOperatorTest.java | 2 +-
 .../java/org/apache/beam/runners/dataflow/AssignWindows.java  | 2 +-
 .../beam/runners/dataflow/BatchStatefulParDoOverrides.java    | 2 +-
 .../org/apache/beam/runners/dataflow/BatchViewOverrides.java  | 2 +-
 .../beam/runners/dataflow/DataflowPipelineTranslator.java     | 2 +-
 .../java/org/apache/beam/runners/dataflow/DataflowRunner.java | 2 +-
 .../beam/runners/dataflow/ReshuffleOverrideFactory.java       | 2 +-
 .../java/org/apache/beam/runners/dataflow/util/DoFnInfo.java  | 2 +-
 .../beam/runners/dataflow/DataflowPipelineTranslatorTest.java | 2 +-
 .../org/apache/beam/runners/dataflow/DataflowRunnerTest.java  | 2 +-
 .../runners/dataflow/transforms/DataflowGroupByKeyTest.java   | 2 +-
 .../beam/runners/dataflow/transforms/DataflowViewTest.java    | 2 +-
 .../java/org/apache/beam/runners/spark/io/CreateStream.java   | 2 +-
 .../spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java    | 2 +-
 .../beam/runners/spark/translation/GroupCombineFunctions.java | 2 +-
 .../beam/runners/spark/translation/MultiDoFnFunction.java     | 2 +-
 .../runners/spark/translation/SparkAbstractCombineFn.java     | 2 +-
 .../beam/runners/spark/translation/SparkGlobalCombineFn.java  | 2 +-
 .../translation/SparkGroupAlsoByWindowViaOutputBufferFn.java  | 2 +-
 .../beam/runners/spark/translation/SparkKeyedCombineFn.java   | 2 +-
 .../runners/spark/translation/StorageLevelPTransform.java     | 2 +-
 .../beam/runners/spark/translation/TransformTranslator.java   | 2 +-
 .../beam/runners/spark/translation/TranslationUtils.java      | 2 +-
 .../translation/streaming/StreamingTransformTranslator.java   | 2 +-
 .../runners/spark/util/SinglePrimitiveOutputPTransform.java   | 2 +-
 .../apache/beam/runners/spark/util/SparkSideInputReader.java  | 2 +-
 sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java | 2 +-
 .../src/main/java/org/apache/beam/sdk/state/TimerSpecs.java   | 2 +-
 .../src/main/java/org/apache/beam/sdk/testing/PAssert.java    | 2 +-
 .../src/main/java/org/apache/beam/sdk/testing/TestStream.java | 2 +-
 .../src/main/java/org/apache/beam/sdk/transforms/Combine.java | 2 +-
 .../src/main/java/org/apache/beam/sdk/transforms/Flatten.java | 2 +-
 .../main/java/org/apache/beam/sdk/transforms/GroupByKey.java  | 2 +-
 .../java/org/apache/beam/sdk/transforms/windowing/Window.java | 4 ++--
 .../main/java/org/apache/beam/sdk/util/AppliedCombineFn.java  | 1 +
 .../main/java/org/apache/beam/sdk/util/IdentityWindowFn.java  | 1 +
 .../main/java/org/apache/beam/sdk/util/PCollectionViews.java  | 1 +
 .../src/main/java/org/apache/beam/sdk/util/Reshuffle.java     | 1 +
 .../src/main/java/org/apache/beam/sdk/values/PCollection.java | 1 -
 .../java/org/apache/beam/sdk/values/PCollectionTuple.java     | 1 -
 .../main/java/org/apache/beam/sdk/values/PCollectionView.java | 1 -
 .../apache/beam/sdk/{util => values}/WindowingStrategy.java   | 2 +-
 .../org/apache/beam/sdk/runners/TransformHierarchyTest.java   | 2 +-
 .../java/org/apache/beam/sdk/runners/TransformTreeTest.java   | 2 +-
 .../org/apache/beam/sdk/testing/PCollectionViewTesting.java   | 2 +-
 .../java/org/apache/beam/sdk/transforms/DoFnTesterTest.java   | 2 +-
 .../java/org/apache/beam/sdk/transforms/GroupByKeyTest.java   | 2 +-
 .../test/java/org/apache/beam/sdk/transforms/ViewTest.java    | 2 +-
 .../org/apache/beam/sdk/transforms/windowing/WindowTest.java  | 4 ++--
 .../src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java | 1 +
 .../java/org/apache/beam/sdk/values/PCollectionTupleTest.java | 1 -
 .../apache/beam/sdk/extensions/protobuf/ByteStringCoder.java  | 1 -
 .../apache/beam/fn/harness/control/ProcessBundleHandler.java  | 2 +-
 .../beam/fn/harness/control/ProcessBundleHandlerTest.java     | 2 +-
 .../org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java   | 2 +-
 128 files changed, 132 insertions(+), 132 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => values}/WindowingStrategy.java (99%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
index 6131bdf41ae9..9d4e9a235a6b 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
@@ -51,10 +51,10 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
index f7242e753a2d..99ad9647322a 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
@@ -61,10 +61,10 @@
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
index 4b73114da4fe..e31d83051dc5 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
@@ -38,10 +38,10 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Assert;
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java
index 1a5c8be4da5c..736b0e736b60 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ParDoTranslatorTest.java
@@ -54,12 +54,12 @@
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
index 907e54dd21c1..0f2fcb7ca281 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
@@ -22,9 +22,9 @@
 import java.io.IOException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * Utility methods for translating {@link PCollection PCollections} to and from Runner API protos.
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
index 5a2140b63ee4..a791a0e20a03 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
@@ -24,12 +24,12 @@
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Create.Values;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * An implementation of {@link Create} that returns a primitive {@link PCollection}.
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index 35af3006d2fe..3e7807221e49 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -34,8 +34,8 @@
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.NameUtils;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /** SDK objects that will be represented at some later point within a {@link Components} object. */
 class SdkComponents {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
index 0c400db1681e..395702f51a3e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
@@ -35,8 +35,8 @@
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.joda.time.Duration;
 
 /** Utilities for working with {@link WindowingStrategy WindowingStrategies}. */
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
index be3755c19ad3..c177c58cb6d6 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
@@ -48,10 +48,10 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.util.VarInt;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index ba935a4e9a0b..cb28c3449295 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -62,7 +62,6 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
@@ -72,6 +71,7 @@
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.junit.Rule;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java
index 00c436d5cb3f..f8d01e9a016f 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java
@@ -26,13 +26,13 @@
 import java.util.Map;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TaggedPValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
index 021f19c347c6..f605e015cc09 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
@@ -38,10 +38,10 @@
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.hamcrest.Matchers;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
index 78ac61c8694e..7296a7789943 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
@@ -29,8 +29,8 @@
 import org.apache.beam.sdk.transforms.windowing.Trigger;
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.junit.Test;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
index c090001ce193..f3dd9a3e20eb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
@@ -29,10 +29,10 @@
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * Static utility methods that provide {@link DoFnRunner} implementations.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java
index 311ed1c70530..a1605534d6a2 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java
@@ -24,7 +24,7 @@
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java
index 34afa5d3d4c3..2342c529342d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java
@@ -22,8 +22,8 @@
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * A general {@link GroupAlsoByWindowsDoFn}. This delegates all of the logic to the
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
index c553dbffed27..4c1fe9516e4c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
@@ -27,9 +27,9 @@
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java
index 694c5ebe056e..fca3c76423d2 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java
@@ -32,9 +32,9 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * An implementation of {@link GroupByKey} built on top of a lower-level {@link GroupByKeyOnly}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
index 872ee3e06657..66385c116ffb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
@@ -27,8 +27,8 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
index 982d693243a3..c45387bacd86 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
@@ -26,7 +26,7 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java
index 8df7ad005aa4..b6e8567c4b8a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/NonEmptyPanes.java
@@ -22,8 +22,8 @@
 import org.apache.beam.sdk.state.ReadableState;
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 
 /**
  * Tracks which windows have non-empty panes. Specifically, which windows have new elements since
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
index 8e2ff7368b1d..b1701ebf3f44 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFn.java
@@ -22,7 +22,7 @@
 import org.apache.beam.sdk.state.Timers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
index 550b9b9769b4..63e8e6d34f48 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
@@ -36,8 +36,8 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
index a949eddfb116..a04006bcedb1 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
@@ -57,10 +57,10 @@
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 2567bbaa3b57..4057790788c1 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -56,9 +56,9 @@
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.joda.time.format.PeriodFormat;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
index 7454dc9248d4..cf9dff233565 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
@@ -39,9 +39,9 @@
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 import org.joda.time.format.PeriodFormat;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
index 74ffeb635339..a4d96392bf2a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
@@ -48,13 +48,13 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
index 346694566084..28a9deef8795 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
@@ -32,7 +32,7 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
index ce15244d795c..64f5d9b165cc 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
@@ -29,7 +29,7 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.util.WindowTracing;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java
index cb8d4943ccf3..a265eade4ffd 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java
@@ -21,7 +21,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java
index 054a2e255c2d..761ffb87024e 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java
@@ -47,11 +47,11 @@
 import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java
index bf784272cee4..88c6ab6e7e1b 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java
@@ -32,7 +32,7 @@
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
index 2b400a40e9a5..381a493d9595 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
@@ -72,10 +72,10 @@
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
index eea30e912e02..6fb0dcb03992 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
@@ -67,11 +67,11 @@
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java
index 335aedecd190..f9e0aaf40845 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java
@@ -29,9 +29,9 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Assert;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index d793584ca446..fae911782d9d 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -45,8 +45,8 @@
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.joda.time.format.PeriodFormat;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
index 0bb25fcfa747..b93ff3a95375 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
@@ -58,13 +58,13 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Rule;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index d7d8261cca5f..2ca8b058761b 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -40,9 +40,9 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
index 51f467ae8171..9a10f53c1bbe 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
@@ -52,8 +52,8 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
index 36050023b399..791615a2a194 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
@@ -29,9 +29,9 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 class DirectGroupByKey<K, V>
     extends ForwardingPTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
index 3cdf351d48f7..75ff1c7e9060 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -45,10 +45,10 @@
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index d00e4080f87e..83548589c1eb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -48,10 +48,10 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
index a3a345f68008..e5a668043203 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
@@ -36,10 +36,10 @@
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 class ParDoEvaluator<InputT> implements TransformEvaluator<InputT> {
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
index b10d66922db8..c72bf61e2c5f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
@@ -44,13 +44,13 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * A {@link PTransformOverrideFactory} that provides overrides for applications of a {@link ParDo}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
index cd459e4c0f5d..1e773c936617 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
@@ -41,8 +41,8 @@
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * An in-process container for {@link PCollectionView PCollectionViews}, which provides methods for
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index fb3a96279b03..2c22caf5606b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -41,11 +41,11 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index ca302b8b11a6..02472b3e69f6 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -48,12 +48,12 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /** A {@link TransformEvaluatorFactory} for stateful {@link ParDo}. */
 final class StatefulParDoEvaluatorFactory<K, InputT, OutputT> implements TransformEvaluatorFactory {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
index b5486c0b0b0d..ea989d754668 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
@@ -40,13 +40,13 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java
index fcefc5ff1df3..ece5a56732fd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java
@@ -26,7 +26,7 @@
 import java.util.concurrent.Executor;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
index 077cd43213c3..3a7fdf3a913b 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
@@ -32,10 +32,10 @@
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;
 import org.junit.Rule;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 63864f0856e2..123e9f233cfa 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -56,11 +56,11 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitorTest.java
index 74e70f880e87..641575742cb1 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitorTest.java
@@ -41,11 +41,11 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 import org.junit.Before;
 import org.junit.Rule;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
index f4de8839c28c..d7c444048165 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
@@ -47,11 +47,11 @@
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 import org.junit.Before;
 import org.junit.Rule;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
index 95bbc36ac145..f6b652d3836f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
@@ -60,13 +60,13 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
index 6875e1a939cb..a36787af86fc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
@@ -40,9 +40,9 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
 import org.apache.beam.sdk.util.PCollectionViews;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
index 8d6e73f54f41..f4a53da28368 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
@@ -32,8 +32,8 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
index f28c8cf3aede..6fffd1a479ec 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
@@ -55,12 +55,12 @@
 import org.apache.beam.sdk.transforms.DoFnTester;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.PCollectionViews;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java
index 6a7689aa3855..205cd3ed9107 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java
@@ -64,12 +64,12 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.FilterFunction;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index bb86cd9dc98f..e96d1c5ec5e4 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -26,11 +26,11 @@
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index 7339c01d9b81..3cf76832447d 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -68,12 +68,12 @@
 import org.apache.beam.sdk.util.AppliedCombineFn;
 import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.common.functions.RichFlatMapFunction;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java
index 6e27057f7ec3..4872a0646061 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java
@@ -26,8 +26,8 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.util.Collector;
 
 /**
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
index 28e1a44d6a3d..9205bce1a83a 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
@@ -30,9 +30,9 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.RichMapPartitionFunction;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.configuration.Configuration;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
index 9ccf0793ffa6..13be91312497 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
@@ -24,9 +24,9 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
 import org.apache.flink.util.Collector;
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
index 4099f52693ba..db12a49e76f2 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
@@ -24,9 +24,9 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.RichGroupCombineFunction;
 import org.apache.flink.util.Collector;
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
index 90dcbff0e8f2..53d71d85c376 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
@@ -24,9 +24,9 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
 import org.apache.flink.util.Collector;
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java
index c317182ffde9..fa95477e1013 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java
@@ -26,9 +26,9 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.RuntimeContext;
 
 /**
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
index 9f000e0f23df..6517bf2c1a9f 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
@@ -40,10 +40,10 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.configuration.Configuration;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java
index 7ee2f6977de6..942bf42f560c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java
@@ -34,8 +34,8 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.util.Collector;
 import org.joda.time.Instant;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java
index 4aacb4a32e28..fb4c67857673 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java
@@ -31,8 +31,8 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.util.Collector;
 import org.joda.time.Instant;
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index 5287b85d0bd6..1844d6d9ce9c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -67,9 +67,9 @@
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
index 1887a992d64f..7d54cfa51d5d 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
@@ -41,9 +41,9 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.streaming.api.operators.InternalTimer;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
index 3899303efd9b..bf64edefbaea 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
@@ -36,10 +36,10 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.streaming.api.operators.InternalTimer;
 
 /**
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
index 7519dbf9d5c1..8382a2d34a3b 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
@@ -53,9 +53,9 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.commons.lang3.SerializationUtils;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.TypeHint;
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
index 80dfa07288c4..79bc0e0b1671 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DoFnOperatorTest.java
@@ -54,10 +54,10 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java
index ffd56c99ecfb..572b00561770 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/AssignWindows.java
@@ -24,8 +24,8 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * A primitive {@link PTransform} that implements the {@link Window#into(WindowFn)}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
index 119c9c9a6440..13ae9ee03b8d 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
@@ -38,13 +38,13 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index b85b315da869..1ff8a3f3affd 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -78,7 +78,6 @@
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
@@ -87,6 +86,7 @@
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * Dataflow batch overrides for {@link CreatePCollectionView}, specialized for different view types.
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index e727433eac08..92f9c9823410 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -91,7 +91,6 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.AppliedCombineFn;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
@@ -100,6 +99,7 @@
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 1a806b9faecb..d5e650ef2a79 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -123,7 +123,6 @@
 import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.ValueWithRecordId;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -133,6 +132,7 @@
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.DateTimeUtils;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
index aa9d9f8c2fee..d33fdfee38ef 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
@@ -31,9 +31,9 @@
 import org.apache.beam.sdk.util.IdentityWindowFn;
 import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.ReshuffleTrigger;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 
 /**
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
index 55c62aea79b7..bd2742f147da 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
@@ -21,9 +21,9 @@
 import java.util.Map;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * Wrapper class holding the necessary information to serialize a {@link DoFn}.
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 986ed5c757a1..48cb79f1731d 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -98,7 +98,6 @@
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.util.GcsPathValidator;
 import org.apache.beam.sdk.util.GcsUtil;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
@@ -106,6 +105,7 @@
 import org.apache.beam.sdk.values.PDone;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index 5aebf29d1033..3d477261fd7a 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -86,10 +86,10 @@
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.util.ReleaseInfo;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Description;
 import org.hamcrest.Matchers;
 import org.hamcrest.TypeSafeMatcher;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
index c9c7806084b9..fc2ff3d284e7 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
@@ -33,11 +33,11 @@
 import org.apache.beam.sdk.transforms.windowing.Sessions;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.NoopPathValidator;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.junit.Before;
 import org.junit.Rule;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
index 455868367ff7..f0dbd6183e60 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
@@ -29,12 +29,12 @@
 import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.NoopPathValidator;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.junit.Before;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
index f2e0bb350639..fdcea99b9165 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
@@ -31,10 +31,10 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 4a2851d87cb5..9bc876018d45 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -50,9 +50,9 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.spark.Partitioner;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
index 6a67ccee6e9d..d19094d1da87 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
@@ -26,8 +26,8 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.spark.HashPartitioner;
 import org.apache.spark.Partitioner;
 import org.apache.spark.api.java.JavaPairRDD;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
index 410b7deb6e40..9bfd2fa3d573 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
@@ -33,9 +33,9 @@
 import org.apache.beam.runners.spark.util.SparkSideInputReader;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.spark.Accumulator;
 import org.apache.spark.api.java.function.PairFlatMapFunction;
 import scala.Tuple2;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
index 7d06d6bf849d..df633b098e4f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
@@ -37,10 +37,10 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 
 /**
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java
index 7d026c68b2c3..d0e90387eb40 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGlobalCombineFn.java
@@ -33,9 +33,9 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
index 9ee52de6262f..a70885bec06f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
@@ -37,9 +37,9 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.spark.Accumulator;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.joda.time.Instant;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java
index 58db8e47d13f..7ac8e7d4904e 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkKeyedCombineFn.java
@@ -33,9 +33,9 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java
index 30b51e66e4df..0ecfa7571ca4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/StorageLevelPTransform.java
@@ -20,8 +20,8 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * Get RDD storage level for the input PCollection (mostly used for testing purpose).
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index d249e78df841..77d2c0e693e6 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -57,12 +57,12 @@
 import org.apache.beam.sdk.util.CombineFnUtil;
 import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.spark.Accumulator;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
index 7b6f9edeecf8..f462e60ac478 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
@@ -38,10 +38,10 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.apache.spark.api.java.function.Function;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index 9af4af2a87c7..08f0e172b638 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -74,12 +74,12 @@
 import org.apache.beam.sdk.util.CombineFnUtil;
 import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.spark.Accumulator;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java
index 7580da77a138..299f5ba97a88 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SinglePrimitiveOutputPTransform.java
@@ -19,10 +19,10 @@
 
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * A {@link PTransform} wrapping another transform.
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java
index d6e1a94a592f..e876e12ec717 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java
@@ -27,10 +27,10 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
index 0e269a2955bc..dbd9b86758d6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
@@ -23,10 +23,10 @@
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
index 9efac69671c7..df4242872905 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
@@ -28,7 +28,7 @@
 public class TimerSpecs {
 
   public static TimerSpec timer(TimeDomain timeDomain) {
-    return new org.apache.beam.sdk.util.AutoValue_TimerSpecs_SimpleTimerSpec(timeDomain);
+    return new AutoValue_TimerSpecs_SimpleTimerSpec(timeDomain);
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
index 85b8c5f3a937..b5d7db5fc204 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
@@ -65,7 +65,6 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -73,6 +72,7 @@
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
index a648767f7202..446c8a5d1974 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
@@ -29,11 +29,11 @@
 import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 9d0e97ab16aa..1be948f779fb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -59,7 +59,6 @@
 import org.apache.beam.sdk.util.NameUtils.NameOverride;
 import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
@@ -69,6 +68,7 @@
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * {@code PTransform}s for combining {@code PCollection} elements
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java
index 7b282b535469..25d9c0519727 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Flatten.java
@@ -21,10 +21,10 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableLikeCoder;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * {@code Flatten<T>} takes multiple {@code PCollection<T>}s bundled
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
index 6a89c5ff3566..7516b255bb56 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
@@ -28,10 +28,10 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * {@code GroupByKey<K, V>} takes a {@code PCollection<KV<K, V>>},
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index d010d1f94ab9..27771912f2dd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -30,10 +30,10 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.joda.time.Duration;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java
index b16aadcf5bdb..71ebba5d8e0f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java
@@ -25,6 +25,7 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * A {@link GlobalCombineFn} with a fixed accumulator coder. This is created from a
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
index d5f21814bbaa..a61e3a6482dc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
@@ -29,6 +29,7 @@
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
index f2052accf0f2..a07bc5e5818b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
@@ -46,6 +46,7 @@
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValueBase;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 
 /**
  * Implementations of {@link PCollectionView} shared across the SDK.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Reshuffle.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Reshuffle.java
index 706e039582b6..887f01103289 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Reshuffle.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Reshuffle.java
@@ -27,6 +27,7 @@
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
index 16223223749c..1095fb8c7558 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
@@ -37,7 +37,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.WindowingStrategy;
 
 /**
  * A {@link PCollection PCollection&lt;T&gt;} is an immutable collection of values of type
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
index 5027df652ddf..d1bb6d7b5667 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
@@ -25,7 +25,6 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
index 71efc09ccdfc..f89041a6124e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
@@ -28,7 +28,6 @@
 import org.apache.beam.sdk.transforms.ViewFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 
 /**
  * A {@link PCollectionView PCollectionView&lt;T&gt;} is an immutable view of a {@link PCollection}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowingStrategy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java
similarity index 99%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowingStrategy.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java
index 14f818afd581..8a773e23f9b5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowingStrategy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.values;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
index e495758c788e..125e15902d17 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
@@ -44,7 +44,6 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
 import org.apache.beam.sdk.transforms.ParDo.SingleOutput;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
@@ -57,6 +56,7 @@
 import org.apache.beam.sdk.values.TaggedPValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.Rule;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java
index 6c3aba272162..e7b680ab6c0a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformTreeTest.java
@@ -40,11 +40,11 @@
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.Sample;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
 import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
index 5f71cabb6148..adf27f8f5279 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
@@ -35,11 +35,11 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValueBase;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
index a980f87b5c74..f74d67344473 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
@@ -35,11 +35,11 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.util.PCollectionViews;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
index 0556199eecfc..50e9c1d4732e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
@@ -55,12 +55,12 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.Reshuffle;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Assert;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
index b3fa2c616979..e72c5401ea81 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
@@ -55,13 +55,13 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 534e2309c468..92f6a9c01457 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -52,11 +52,11 @@
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java
index eae465c73753..71bfdb5c7585 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java
@@ -43,6 +43,7 @@
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Rule;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java
index b704d07f0ad2..58e2bbdc859a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java
@@ -40,7 +40,6 @@
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SimpleFunction;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.hamcrest.Matchers;
 import org.junit.Rule;
diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
index 30570508cfc3..d4e6f63f116c 100644
--- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
+++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
@@ -23,7 +23,6 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.util.VarInt;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
index f29aeb9f3ce7..46bd6d437e42 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
@@ -61,8 +61,8 @@
 import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 6d5e230b9c20..3e06369320af 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -75,9 +75,9 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index b893ad5a5189..38da2d94c112 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -123,13 +123,13 @@
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;

From 1d4b1ed49ce2212d961ad37325a3bfce8964f673 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 19:36:45 -0700
Subject: [PATCH 055/387] Move PathValidator adjacent to other filesystem bits

---
 .../beam/runners/dataflow/DataflowRunner.java |  2 +-
 .../beam/sdk/io/fs/NoopPathValidator.java     | 52 +++++++++++++++++++
 .../sdk/{util => io/fs}/PathValidator.java    |  9 ++--
 .../beam/sdk/util/NoopPathValidator.java      |  4 +-
 .../extensions/gcp/options/GcpOptions.java    |  2 +-
 .../extensions/gcp/options/GcsOptions.java    |  2 +-
 .../beam/sdk/util/GcsPathValidator.java       |  1 +
 7 files changed, 65 insertions(+), 7 deletions(-)
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/NoopPathValidator.java
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => io/fs}/PathValidator.java (87%)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index d5e650ef2a79..ce824c66843b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -88,6 +88,7 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.io.fs.PathValidator;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
@@ -118,7 +119,6 @@
 import org.apache.beam.sdk.util.InstanceBuilder;
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.NameUtils;
-import org.apache.beam.sdk.util.PathValidator;
 import org.apache.beam.sdk.util.ReleaseInfo;
 import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.ValueWithRecordId;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/NoopPathValidator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/NoopPathValidator.java
new file mode 100644
index 000000000000..d5be8f0fc5fb
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/NoopPathValidator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.fs;
+
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.options.PipelineOptions;
+
+/**
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
+ *
+ * <p>Noop implementation of {@link PathValidator}. All paths are allowed and returned unchanged.
+ */
+@Internal
+public class NoopPathValidator implements PathValidator {
+
+  private NoopPathValidator() {
+  }
+
+  public static PathValidator fromOptions(
+      @SuppressWarnings("unused") PipelineOptions options) {
+    return new NoopPathValidator();
+  }
+
+  @Override
+  public void validateInputFilePatternSupported(String filepattern) {}
+
+  @Override
+  public void validateOutputFilePrefixSupported(String filePrefix) {}
+
+  @Override
+  public void validateOutputResourceSupported(ResourceId resourceId) {}
+
+  @Override
+  public String verifyPath(String path) {
+    return path;
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PathValidator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/PathValidator.java
similarity index 87%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/PathValidator.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/PathValidator.java
index e69648b0d2da..b88a33e75727 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PathValidator.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/PathValidator.java
@@ -15,13 +15,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.io.fs;
 
-import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.annotations.Internal;
 
 /**
- * Interface for controlling validation of paths.
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
+ *
+ * <p>Interface for controlling validation of paths.
  */
+@Internal
 public interface PathValidator {
   /**
    * Validate that a file pattern is conforming.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
index 1f3f5a848dd2..0015e5945767 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
@@ -17,12 +17,14 @@
  */
 package org.apache.beam.sdk.util;
 
+import org.apache.beam.sdk.io.fs.PathValidator;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
 
 /**
- * Noop implementation of {@link PathValidator}. All paths are allowed and returned unchanged.
+ * @deprecated use {@link org.apache.beam.sdk.io.fs.NoopPathValidator}.
  */
+@Deprecated
 public class NoopPathValidator implements PathValidator {
 
   private NoopPathValidator() {
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index 445f00fd5c5a..126b7952d8d2 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -48,13 +48,13 @@
 import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory;
 import org.apache.beam.sdk.extensions.gcp.auth.GcpCredentialFactory;
 import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer;
+import org.apache.beam.sdk.io.fs.PathValidator;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.InstanceBuilder;
-import org.apache.beam.sdk.util.PathValidator;
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
 import org.apache.beam.sdk.util.Transport;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java
index 954092c145c6..78e233e967c1 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java
@@ -26,6 +26,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.fs.PathValidator;
 import org.apache.beam.sdk.options.ApplicationNameOptions;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
@@ -35,7 +36,6 @@
 import org.apache.beam.sdk.util.GcsPathValidator;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.InstanceBuilder;
-import org.apache.beam.sdk.util.PathValidator;
 
 /**
  * Options used to configure Google Cloud Storage.
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
index 9ad4152ca090..c4e557bd41de 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
+import org.apache.beam.sdk.io.fs.PathValidator;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;

From 6542eafcf3a84105d2716cd0200d40ccdf764472 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 19:46:59 -0700
Subject: [PATCH 056/387] Move SideInputReader to runners/core-java

---
 .../runners/apex/ApexRunnerRegistrar.java     |  1 -
 .../beam/runners/apex/ApexRunnerResult.java   |  2 -
 .../beam/runners/apex/ApexYarnLauncher.java   |  2 -
 .../translation/ReadUnboundedTranslator.java  |  1 -
 .../apex/translation/TransformTranslator.java |  1 -
 .../operators/ApexGroupByKeyOperator.java     |  2 +-
 .../operators/ApexParDoOperator.java          |  4 +-
 .../ApexReadUnboundedInputOperator.java       |  2 -
 .../utils/CoderAdapterStreamCodec.java        |  2 -
 .../utils/ValueAndCoderKryoSerializable.java  |  2 -
 .../apex/translation/utils/ValuesSource.java  |  2 -
 .../runners/apex/ApexYarnLauncherTest.java    |  2 -
 .../apex/examples/UnboundedTextSource.java    |  2 -
 .../runners/apex/examples/WordCountTest.java  |  2 -
 .../ApexGroupByKeyOperatorTest.java           |  2 -
 .../translation/utils/CollectionSource.java   |  2 -
 .../apache/beam/runners/core/DoFnRunners.java |  2 -
 .../runners/core/GlobalCombineFnRunner.java   |  1 -
 .../runners/core/GlobalCombineFnRunners.java  | 41 +++++++++++++++----
 .../GroupAlsoByWindowViaWindowSetNewDoFn.java |  1 -
 .../runners/core}/NullSideInputReader.java    |  2 +-
 ...oundedSplittableProcessElementInvoker.java |  1 -
 .../beam/runners/core/ProcessFnRunner.java    |  1 -
 .../core}/ReadyCheckingSideInputReader.java   |  2 +-
 .../runners/core/ReduceFnContextFactory.java  |  1 -
 .../beam/runners/core/ReduceFnRunner.java     |  1 -
 .../beam/runners/core/SideInputHandler.java   |  1 -
 .../beam/runners/core}/SideInputReader.java   |  2 +-
 .../beam/runners/core/SimpleDoFnRunner.java   |  1 -
 .../runners/core/SimpleOldDoFnRunner.java     |  1 -
 .../SimplePushbackSideInputDoFnRunner.java    |  1 -
 .../core/UnsupportedSideInputReader.java      |  1 -
 .../core/WindowingInternalsAdapters.java      |  1 -
 ...edSplittableProcessElementInvokerTest.java |  1 -
 .../beam/runners/core/ReduceFnRunnerTest.java |  1 -
 .../beam/runners/core/ReduceFnTester.java     |  2 -
 .../runners/core/SimpleDoFnRunnerTest.java    |  1 -
 ...SimplePushbackSideInputDoFnRunnerTest.java |  1 -
 .../runners/core/SplittableParDoTest.java     |  1 -
 .../runners/core/StatefulDoFnRunnerTest.java  |  1 -
 .../runners/direct/EvaluationContext.java     |  4 +-
 .../beam/runners/direct/ParDoEvaluator.java   |  2 +-
 .../runners/direct/SideInputContainer.java    |  4 +-
 ...ttableProcessElementsEvaluatorFactory.java |  2 +-
 .../runners/direct/EvaluationContextTest.java |  2 +-
 .../runners/direct/ParDoEvaluatorTest.java    |  2 +-
 .../direct/SideInputContainerTest.java        |  4 +-
 .../StatefulParDoEvaluatorFactoryTest.java    |  2 +-
 .../functions/AbstractFlinkCombineRunner.java |  2 +-
 .../functions/FlinkSideInputReader.java       |  2 +-
 .../functions/HashingFlinkCombineRunner.java  |  2 +-
 .../functions/SortingFlinkCombineRunner.java  |  2 +-
 .../wrappers/streaming/DoFnOperator.java      |  4 +-
 .../runners/spark/SparkRunnerDebugger.java    |  1 -
 .../spark/aggregators/NamedAggregators.java   |  2 -
 .../runners/spark/coders/CoderHelpers.java    |  1 -
 .../spark/coders/StatelessJavaSerializer.java |  1 -
 .../spark/io/SparkUnboundedSource.java        |  1 -
 .../spark/metrics/AggregatorMetric.java       |  1 -
 .../spark/metrics/SparkBeamMetricSource.java  |  1 -
 .../spark/stateful/StateSpecFunctions.java    |  1 -
 .../translation/SparkAbstractCombineFn.java   |  2 +-
 .../spark/translation/TranslationUtils.java   |  1 -
 .../spark/util/GlobalWatermarkHolder.java     |  1 -
 .../spark/util/SparkSideInputReader.java      |  2 +-
 .../beam/sdk/util/CombineContextFactory.java  | 25 -----------
 .../harness/control/ProcessBundleHandler.java |  2 +-
 67 files changed, 60 insertions(+), 116 deletions(-)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => runners/core-java/src/main/java/org/apache/beam/runners/core}/NullSideInputReader.java (98%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => runners/core-java/src/main/java/org/apache/beam/runners/core}/ReadyCheckingSideInputReader.java (97%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => runners/core-java/src/main/java/org/apache/beam/runners/core}/SideInputReader.java (97%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java
index aa6ef4546b23..8cde692f8c15 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java
@@ -20,7 +20,6 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
-
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
 import org.apache.beam.sdk.runners.PipelineRunner;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java
index 41fdb7577dad..cc24ddd82a94 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerResult.java
@@ -18,9 +18,7 @@
 package org.apache.beam.runners.apex;
 
 import com.datatorrent.api.DAG;
-
 import java.io.IOException;
-
 import org.apache.apex.api.Launcher.AppHandle;
 import org.apache.apex.api.Launcher.ShutdownMode;
 import org.apache.beam.sdk.Pipeline;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
index b84144cd251c..18d8e940ee60 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
@@ -25,7 +25,6 @@
 import com.datatorrent.api.StreamingApplication;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Sets;
-
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -56,7 +55,6 @@
 import java.util.Set;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
-
 import org.apache.apex.api.EmbeddedAppLauncher;
 import org.apache.apex.api.Launcher;
 import org.apache.apex.api.Launcher.AppHandle;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ReadUnboundedTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ReadUnboundedTranslator.java
index b3034ac472e2..168cbf5c221c 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ReadUnboundedTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ReadUnboundedTranslator.java
@@ -19,7 +19,6 @@
 package org.apache.beam.runners.apex.translation;
 
 import com.datatorrent.api.InputOperator;
-
 import org.apache.beam.runners.apex.translation.operators.ApexReadUnboundedInputOperator;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.UnboundedSource;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java
index eb810521c98a..49ff49b4dbf3 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java
@@ -20,7 +20,6 @@
 
 
 import java.io.Serializable;
-
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
index 9d4e9a235a6b..85836ad2acbf 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
@@ -34,6 +34,7 @@
 import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.SerializablePipelineOptions;
+import org.apache.beam.runners.core.NullSideInputReader;
 import org.apache.beam.runners.core.OutputWindowedValue;
 import org.apache.beam.runners.core.ReduceFnRunner;
 import org.apache.beam.runners.core.StateInternalsFactory;
@@ -49,7 +50,6 @@
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
index 99ad9647322a..8c516b1eb9ca 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
@@ -43,8 +43,10 @@
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItemCoder;
+import org.apache.beam.runners.core.NullSideInputReader;
 import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
 import org.apache.beam.runners.core.SideInputHandler;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StatefulDoFnRunner;
@@ -56,8 +58,6 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.util.NullSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java
index ac28c2a8e338..1549560f56fe 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexReadUnboundedInputOperator.java
@@ -26,9 +26,7 @@
 import com.esotericsoftware.kryo.serializers.FieldSerializer.Bind;
 import com.esotericsoftware.kryo.serializers.JavaSerializer;
 import com.google.common.base.Throwables;
-
 import java.io.IOException;
-
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple.DataTuple;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/CoderAdapterStreamCodec.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/CoderAdapterStreamCodec.java
index d08e76f09657..f6ce1d075b22 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/CoderAdapterStreamCodec.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/CoderAdapterStreamCodec.java
@@ -19,12 +19,10 @@
 
 import com.datatorrent.api.StreamCodec;
 import com.datatorrent.netlet.util.Slice;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.Serializable;
-
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
 
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValueAndCoderKryoSerializable.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValueAndCoderKryoSerializable.java
index 395ad1f3945d..2a72f1719346 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValueAndCoderKryoSerializable.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValueAndCoderKryoSerializable.java
@@ -22,9 +22,7 @@
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 import com.esotericsoftware.kryo.serializers.JavaSerializer;
-
 import java.io.IOException;
-
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
 
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java
index 62c92a004892..41f027faa96d 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ValuesSource.java
@@ -24,9 +24,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
-
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.IterableCoder;
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java
index 6ffb0919dc42..68ec2ea79dcf 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexYarnLauncherTest.java
@@ -26,7 +26,6 @@
 import com.datatorrent.api.Context.DAGContext;
 import com.datatorrent.api.DAG;
 import com.datatorrent.api.StreamingApplication;
-
 import java.io.File;
 import java.net.URI;
 import java.nio.file.FileSystem;
@@ -37,7 +36,6 @@
 import java.util.Map;
 import java.util.Properties;
 import java.util.jar.JarFile;
-
 import org.apache.apex.api.EmbeddedAppLauncher;
 import org.apache.apex.api.Launcher;
 import org.apache.apex.api.Launcher.AppHandle;
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java
index abe97f6de049..c590a2e9783c 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/UnboundedTextSource.java
@@ -23,9 +23,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.NoSuchElementException;
-
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.UnboundedSource;
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java
index 83af61b028a6..e76096ef78d1 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/examples/WordCountTest.java
@@ -18,11 +18,9 @@
 package org.apache.beam.runners.apex.examples;
 
 import com.google.common.collect.Sets;
-
 import java.io.File;
 import java.util.HashSet;
 import java.util.concurrent.ConcurrentHashMap;
-
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.runners.apex.ApexRunner;
 import org.apache.beam.runners.apex.ApexRunnerResult;
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
index e31d83051dc5..206b43001734 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/ApexGroupByKeyOperatorTest.java
@@ -20,9 +20,7 @@
 import com.datatorrent.api.Sink;
 import com.datatorrent.lib.util.KryoCloneUtils;
 import com.google.common.collect.Lists;
-
 import java.util.List;
-
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.runners.apex.TestApexRunner;
 import org.apache.beam.runners.apex.translation.operators.ApexGroupByKeyOperator;
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java
index 92812b4f4f5d..288aadedb46c 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/utils/CollectionSource.java
@@ -25,9 +25,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
-
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
index f3dd9a3e20eb..ee3aefa1054b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
@@ -26,8 +26,6 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunner.java
index 5325ba63d1e1..4c312b43491b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunner.java
@@ -22,7 +22,6 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SideInputReader;
 
 /**
  * An interface that runs a {@link GlobalCombineFn} with unified APIs.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java
index d45b5031ee91..d98bac83d496 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java
@@ -22,10 +22,10 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
+import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.CombineContextFactory;
-import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.values.PCollectionView;
 
 /**
  * Static utility methods that provide {@link GlobalCombineFnRunner} implementations for different
@@ -46,6 +46,33 @@ public static <InputT, AccumT, OutputT> GlobalCombineFnRunner<InputT, AccumT, Ou
     }
   }
 
+  /**
+   * Returns a {@code Combine.Context} from {@code PipelineOptions}, {@code SideInputReader}, and
+   * the main input window.
+   */
+  private static CombineWithContext.Context createFromComponents(
+      final PipelineOptions options,
+      final SideInputReader sideInputReader,
+      final BoundedWindow mainInputWindow) {
+    return new CombineWithContext.Context() {
+      @Override
+      public PipelineOptions getPipelineOptions() {
+        return options;
+      }
+
+      @Override
+      public <T> T sideInput(PCollectionView<T> view) {
+        if (!sideInputReader.contains(view)) {
+          throw new IllegalArgumentException("calling sideInput() with unknown view");
+        }
+
+        BoundedWindow sideInputWindow =
+            view.getWindowMappingFn().getSideInputWindow(mainInputWindow);
+        return sideInputReader.get(view, sideInputWindow);
+      }
+    };
+  }
+
   /**
    * An implementation of {@link GlobalCombineFnRunner} with {@link CombineFn}.
    *
@@ -136,7 +163,7 @@ public AccumT createAccumulator(
         SideInputReader sideInputReader,
         Collection<? extends BoundedWindow> windows) {
       return combineFnWithContext.createAccumulator(
-          CombineContextFactory.createFromComponents(
+          createFromComponents(
               options, sideInputReader, Iterables.getOnlyElement(windows)));
     }
 
@@ -150,7 +177,7 @@ public AccumT addInput(
       return combineFnWithContext.addInput(
           accumulator,
           input,
-          CombineContextFactory.createFromComponents(
+          createFromComponents(
               options, sideInputReader, Iterables.getOnlyElement(windows)));
     }
 
@@ -162,7 +189,7 @@ public AccumT mergeAccumulators(
         Collection<? extends BoundedWindow> windows) {
       return combineFnWithContext.mergeAccumulators(
           accumulators,
-          CombineContextFactory.createFromComponents(
+          createFromComponents(
               options, sideInputReader, Iterables.getOnlyElement(windows)));
     }
 
@@ -174,7 +201,7 @@ public OutputT extractOutput(
         Collection<? extends BoundedWindow> windows) {
       return combineFnWithContext.extractOutput(
           accumulator,
-          CombineContextFactory.createFromComponents(
+          createFromComponents(
               options, sideInputReader, Iterables.getOnlyElement(windows)));
     }
 
@@ -186,7 +213,7 @@ public AccumT compact(
         Collection<? extends BoundedWindow> windows) {
       return combineFnWithContext.compact(
           accumulator,
-          CombineContextFactory.createFromComponents(
+          createFromComponents(
               options, sideInputReader, Iterables.getOnlyElement(windows)));
     }
   }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
index 4c1fe9516e4c..5b82d1fdfda4 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
@@ -24,7 +24,6 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NullSideInputReader.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/NullSideInputReader.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/NullSideInputReader.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/NullSideInputReader.java
index 3da2dc17b401..786ed41a80ef 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NullSideInputReader.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/NullSideInputReader.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.runners.core;
 
 import com.google.common.collect.Sets;
 import java.util.Collections;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index 16bdfa361c0d..2db6531e050c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -36,7 +36,6 @@
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
index 7cbf0d214053..61f413fd8cac 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
@@ -28,7 +28,6 @@
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Instant;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReadyCheckingSideInputReader.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReadyCheckingSideInputReader.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReadyCheckingSideInputReader.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/ReadyCheckingSideInputReader.java
index cb38a55a0da4..8d1f0e277be1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReadyCheckingSideInputReader.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReadyCheckingSideInputReader.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.util;
+package org.apache.beam.runners.core;
 
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
index 63e8e6d34f48..bd327f312378 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java
@@ -35,7 +35,6 @@
 import org.apache.beam.sdk.state.Timers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
index a04006bcedb1..d2ed835ba463 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
@@ -54,7 +54,6 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
index af750104e6dc..539b9f08c67c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java
@@ -32,7 +32,6 @@
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SideInputReader.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputReader.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/SideInputReader.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputReader.java
index e81c704cf196..7d1b829710f2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SideInputReader.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputReader.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.runners.core;
 
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 4057790788c1..aab34a51f5eb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -52,7 +52,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
index cf9dff233565..2a0b688142b6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
@@ -35,7 +35,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java
index 36a04d9e2f04..3f77f7d988f0 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java
@@ -25,7 +25,6 @@
 import java.util.Set;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Instant;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java
index 4230f8c75c4c..c1c717952b05 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java
@@ -18,7 +18,6 @@
 package org.apache.beam.runners.core;
 
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.values.PCollectionView;
 
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java
index 1b36bf9c4872..4a5844562980 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java
@@ -20,7 +20,6 @@
 import java.util.Collection;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.joda.time.Instant;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java
index 541e2383913e..a2f6acc9cdeb 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java
@@ -33,7 +33,6 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.joda.time.Duration;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
index 381a493d9595..3dd98e0b7cf7 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
@@ -70,7 +70,6 @@
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
index 6fb0dcb03992..7de8f3b28dfd 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
@@ -62,9 +62,7 @@
 import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.AppliedCombineFn;
-import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index fae911782d9d..abefd1c488b7 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -42,7 +42,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java
index dabc4f013b86..79bf0b29f797 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java
@@ -39,7 +39,6 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.IdentitySideInputWindowFn;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
index b93ff3a95375..be4cf08b662d 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
@@ -56,7 +56,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index 2ca8b058761b..76351e40cad3 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -38,7 +38,6 @@
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
index 75ff1c7e9060..7b6461103477 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -32,6 +32,8 @@
 import java.util.concurrent.ConcurrentMap;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.ExecutionContext;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
 import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
@@ -42,8 +44,6 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.Trigger;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
index e5a668043203..4cfd16fec901 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
@@ -26,6 +26,7 @@
 import org.apache.beam.runners.core.DoFnRunners;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
 import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
 import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
@@ -33,7 +34,6 @@
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
index 1e773c936617..380dc65b0090 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
@@ -35,11 +35,11 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.PCollectionViewWindow;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.WindowingStrategy;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index 2c22caf5606b..44f2e853f97c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -27,6 +27,7 @@
 import org.apache.beam.runners.core.OutputAndTimeBoundedSplittableProcessElementInvoker;
 import org.apache.beam.runners.core.OutputWindowedValue;
 import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
 import org.apache.beam.runners.core.SplittableParDo;
 import org.apache.beam.runners.core.SplittableParDo.ProcessFn;
 import org.apache.beam.runners.core.StateInternals;
@@ -39,7 +40,6 @@
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 123e9f233cfa..077bb6865cfc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -31,6 +31,7 @@
 import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.StateNamespaces;
 import org.apache.beam.runners.core.StateTag;
 import org.apache.beam.runners.core.StateTags;
@@ -54,7 +55,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index ef8add968180..88fd5d2f159d 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -30,6 +30,7 @@
 import java.util.Collection;
 import java.util.List;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -44,7 +45,6 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.IdentitySideInputWindowFn;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
index d7c444048165..d4ca9fdd76b7 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
@@ -32,6 +32,8 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -44,8 +46,6 @@
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
 import org.apache.beam.sdk.util.PCollectionViews;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
index f6b652d3836f..f1ba57cf8c49 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
@@ -35,6 +35,7 @@
 import java.util.List;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItems;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StateNamespace;
 import org.apache.beam.runners.core.StateNamespaces;
@@ -58,7 +59,6 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java
index 4872a0646061..2ae7833764e1 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java
@@ -21,10 +21,10 @@
 import java.util.Collection;
 import org.apache.beam.runners.core.GlobalCombineFnRunner;
 import org.apache.beam.runners.core.GlobalCombineFnRunners;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.CombineFnBase;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.WindowingStrategy;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java
index fa95477e1013..f2752907b70a 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java
@@ -23,8 +23,8 @@
 import java.util.HashMap;
 import java.util.Map;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java
index 942bf42f560c..feb8b3931865 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java
@@ -27,12 +27,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.WindowingStrategy;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java
index fb4c67857673..026a35c85cb1 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java
@@ -23,13 +23,13 @@
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.WindowingStrategy;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index 1844d6d9ce9c..f35ba7a0b086 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -34,8 +34,10 @@
 import org.apache.beam.runners.core.DoFnRunners;
 import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetNewDoFn;
+import org.apache.beam.runners.core.NullSideInputReader;
 import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
 import org.apache.beam.runners.core.SideInputHandler;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StateNamespace;
@@ -64,8 +66,6 @@
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.NullSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
index 8d47e1a31e82..11c52c7a2ff3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
@@ -20,7 +20,6 @@
 
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
-
 import org.apache.beam.runners.spark.translation.EvaluationContext;
 import org.apache.beam.runners.spark.translation.SparkPipelineTranslator;
 import org.apache.beam.runners.spark.translation.TransformTranslator;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
index cf6c9ade8e13..c836ca5801b8 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
@@ -22,14 +22,12 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
-
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.util.Map;
 import java.util.TreeMap;
-
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java
index 9c46ecf379db..85e3572b1de4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java
@@ -32,7 +32,6 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.spark.api.java.function.Function;
 import org.apache.spark.api.java.function.PairFunction;
-
 import scala.Tuple2;
 
 /**
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/StatelessJavaSerializer.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/StatelessJavaSerializer.java
index 01b3b93c6869..0cf4951dc4f7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/StatelessJavaSerializer.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/StatelessJavaSerializer.java
@@ -23,7 +23,6 @@
 import com.esotericsoftware.kryo.Serializer;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.ObjectInputStream;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
index 6b34590be1e6..0388f6c1f1e8 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
@@ -58,7 +58,6 @@
 import org.apache.spark.streaming.dstream.DStream;
 import org.apache.spark.streaming.scheduler.StreamInputInfo;
 import org.joda.time.Instant;
-
 import scala.Tuple2;
 import scala.runtime.BoxedUnit;
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/AggregatorMetric.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/AggregatorMetric.java
index 271cc6bff867..450bc95939fd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/AggregatorMetric.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/AggregatorMetric.java
@@ -19,7 +19,6 @@
 package org.apache.beam.runners.spark.metrics;
 
 import com.codahale.metrics.Metric;
-
 import org.apache.beam.runners.spark.aggregators.NamedAggregators;
 
 /**
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java
index 9cab66dae2f6..5c6fc241265c 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java
@@ -19,7 +19,6 @@
 package org.apache.beam.runners.spark.metrics;
 
 import com.codahale.metrics.MetricRegistry;
-
 import org.apache.spark.metrics.source.Source;
 
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
index 30ee639b2b67..d8d52c425ecb 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
@@ -47,7 +47,6 @@
 import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
 import scala.Option;
 import scala.Tuple2;
 import scala.runtime.AbstractFunction3;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
index df633b098e4f..315f7fb18ddd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAbstractCombineFn.java
@@ -29,13 +29,13 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.spark.util.SideInputBroadcast;
 import org.apache.beam.runners.spark.util.SparkSideInputReader;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
index f462e60ac478..993062c048d4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
@@ -50,7 +50,6 @@
 import org.apache.spark.api.java.function.VoidFunction;
 import org.apache.spark.streaming.api.java.JavaDStream;
 import org.apache.spark.streaming.api.java.JavaPairDStream;
-
 import scala.Tuple2;
 
 /** A set of utilities to help translating Beam transformations into Spark transformations. */
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
index 212f974211b8..8b384d8e2ab0 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
@@ -26,7 +26,6 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
-
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.broadcast.Broadcast;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java
index e876e12ec717..6c91088c2c3f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java
@@ -24,8 +24,8 @@
 import com.google.common.collect.Iterables;
 import java.util.Map;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SideInputReader;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java
index f93cb0b50200..b72fd8280ea4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CombineContextFactory.java
@@ -20,7 +20,6 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.StateContext;
 import org.apache.beam.sdk.transforms.CombineWithContext.Context;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
@@ -64,28 +63,4 @@ public <T> T sideInput(PCollectionView<T> view) {
     };
   }
 
-  /**
-   * Returns a {@code Combine.Context} from {@code PipelineOptions}, {@code SideInputReader},
-   * and the main input window.
-   */
-  public static Context createFromComponents(final PipelineOptions options,
-      final SideInputReader sideInputReader, final BoundedWindow mainInputWindow) {
-    return new Context() {
-      @Override
-      public PipelineOptions getPipelineOptions() {
-        return options;
-      }
-
-      @Override
-      public <T> T sideInput(PCollectionView<T> view) {
-        if (!sideInputReader.contains(view)) {
-          throw new IllegalArgumentException("calling sideInput() with unknown view");
-        }
-
-        BoundedWindow sideInputWindow =
-            view.getWindowMappingFn().getSideInputWindow(mainInputWindow);
-        return sideInputReader.get(view, sideInputWindow);
-      }
-    };
-  }
 }
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
index 46bd6d437e42..fd9f0dfb534d 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java
@@ -53,12 +53,12 @@
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.runners.core.DoFnRunners;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
+import org.apache.beam.runners.core.NullSideInputReader;
 import org.apache.beam.runners.dataflow.util.DoFnInfo;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.util.NullSideInputReader;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.TupleTag;

From e0b3f8064b97d8678e75bf6ba25244bca31e6a7d Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 19:57:02 -0700
Subject: [PATCH 057/387] Move Reshuffle and ReshuffleTrigger out of util

These are two deprecated, but regularly used, transforms and triggers, to the
point where the Runner API proto even includes something akin to the
ReshuffleTrigger.

They are moved to where they "belong" and marked deprecated.
---
 .../runners/core/construction/Triggers.java   |  2 +-
 .../ReshuffleTriggerStateMachine.java         |  2 +-
 .../flink/FlinkBatchTransformTranslators.java |  2 +-
 .../FlinkStreamingTransformTranslators.java   |  2 +-
 .../beam/runners/dataflow/DataflowRunner.java |  2 +-
 .../dataflow/ReshuffleOverrideFactory.java    |  4 +--
 .../translation/GroupCombineFunctions.java    |  3 ++-
 .../translation/TransformTranslator.java      |  2 +-
 .../StreamingTransformTranslator.java         |  2 +-
 .../org/apache/beam/sdk/io/FileSystems.java   |  1 -
 .../{util => transforms}/ReifyTimestamps.java |  7 ++---
 .../sdk/{util => transforms}/Reshuffle.java   | 27 ++++++++++---------
 .../windowing}/ReshuffleTrigger.java          | 16 +++++++----
 .../beam/sdk/transforms/GroupByKeyTest.java   |  1 -
 .../ReifyTimestampsTest.java                  |  7 +----
 .../{util => transforms}/ReshuffleTest.java   |  7 +----
 .../windowing}/ReshuffleTriggerTest.java      |  4 +--
 .../extensions/sorter/ExternalSorterTest.java |  1 -
 .../apache/beam/sdk/io/common/HashingFn.java  |  2 --
 .../beam/sdk/io/gcp/bigquery/BatchLoads.java  |  2 +-
 .../sdk/io/gcp/bigquery/BigQueryHelpers.java  |  2 --
 .../io/gcp/bigquery/BigQueryQuerySource.java  |  1 -
 .../sdk/io/gcp/bigquery/CalculateSchemas.java |  1 -
 .../sdk/io/gcp/bigquery/CreateTables.java     |  1 -
 .../io/gcp/bigquery/StreamingWriteTables.java |  2 +-
 .../sdk/io/gcp/bigquery/TableDestination.java |  1 -
 .../sdk/io/gcp/bigquery/WritePartition.java   |  1 -
 .../beam/sdk/io/gcp/bigquery/WriteRename.java |  1 -
 .../beam/sdk/io/gcp/bigquery/WriteTables.java |  1 -
 .../io/gcp/bigquery/FakeBigQueryServices.java |  2 --
 .../sdk/io/gcp/bigquery/FakeJobService.java   |  3 ---
 .../sdk/io/gcp/bigquery/TableContainer.java   |  1 -
 .../ConfigurableEmployeeInputFormat.java      |  1 -
 .../inputformat/EmployeeInputFormat.java      |  1 -
 .../ReuseObjectsEmployeeInputFormat.java      |  1 -
 .../inputformat/TestEmployeeDataSet.java      |  2 --
 .../inputformat/HIFIOWithElasticTest.java     |  1 -
 .../HIFIOWithEmbeddedCassandraTest.java       |  2 --
 .../integration/tests/HIFIOCassandraIT.java   |  2 --
 .../integration/tests/HIFIOElasticIT.java     |  1 -
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java   |  2 --
 .../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java |  1 -
 .../org/apache/beam/sdk/io/jms/JmsRecord.java |  1 -
 .../sdk/io/kafka/KafkaCheckpointMark.java     |  2 --
 .../apache/beam/sdk/io/kafka/KafkaRecord.java |  1 -
 .../CoderBasedKafkaDeserializer.java          |  1 -
 .../CoderBasedKafkaSerializer.java            |  1 -
 .../serialization/InstantDeserializer.java    |  1 -
 .../serialization/InstantSerializer.java      |  1 -
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java  |  3 ---
 .../sdk/io/mongodb/MongoDBGridFSIOTest.java   |  3 ---
 .../beam/sdk/io/mongodb/MongoDbIOTest.java    |  4 ---
 .../org/apache/beam/sdk/io/mqtt/MqttIO.java   |  3 ---
 53 files changed, 44 insertions(+), 104 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => transforms}/ReifyTimestamps.java (92%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => transforms}/Reshuffle.java (79%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => transforms/windowing}/ReshuffleTrigger.java (75%)
 rename sdks/java/core/src/test/java/org/apache/beam/sdk/{util => transforms}/ReifyTimestampsTest.java (93%)
 rename sdks/java/core/src/test/java/org/apache/beam/sdk/{util => transforms}/ReshuffleTest.java (97%)
 rename sdks/java/core/src/test/java/org/apache/beam/sdk/{util => transforms/windowing}/ReshuffleTriggerTest.java (90%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
index 5e735715b3ee..df6c9ed1fa04 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
@@ -40,10 +40,10 @@
 import org.apache.beam.sdk.transforms.windowing.Never.NeverTrigger;
 import org.apache.beam.sdk.transforms.windowing.OrFinallyTrigger;
 import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.ReshuffleTrigger;
 import org.apache.beam.sdk.transforms.windowing.TimestampTransform;
 import org.apache.beam.sdk.transforms.windowing.Trigger;
 import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger;
-import org.apache.beam.sdk.util.ReshuffleTrigger;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ReshuffleTriggerStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ReshuffleTriggerStateMachine.java
index 8a2b736b629d..61ede342d921 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ReshuffleTriggerStateMachine.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ReshuffleTriggerStateMachine.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.runners.core.triggers;
 
-import org.apache.beam.sdk.util.Reshuffle;
+import org.apache.beam.sdk.transforms.Reshuffle;
 
 /**
  * The trigger used with {@link Reshuffle} which triggers on every element
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java
index 205cd3ed9107..3689698c1c61 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java
@@ -53,6 +53,7 @@
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.join.RawUnionValue;
 import org.apache.beam.sdk.transforms.join.UnionCoder;
@@ -62,7 +63,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index 3cf76832447d..615eaea29735 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -56,6 +56,7 @@
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.join.RawUnionValue;
 import org.apache.beam.sdk.transforms.join.UnionCoder;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
@@ -66,7 +67,6 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.AppliedCombineFn;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index ce824c66843b..7123316f436e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -110,6 +110,7 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.transforms.View;
@@ -120,7 +121,6 @@
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.util.ReleaseInfo;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.ValueWithRecordId;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
index d33fdfee38ef..5814efd0ce47 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
@@ -26,11 +26,11 @@
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.ReshuffleTrigger;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.IdentityWindowFn;
-import org.apache.beam.sdk.util.Reshuffle;
-import org.apache.beam.sdk.util.ReshuffleTrigger;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.WindowingStrategy;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
index d19094d1da87..1fa46b5b634e 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
@@ -24,6 +24,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
 import org.apache.beam.sdk.values.KV;
@@ -223,7 +224,7 @@ public static <InputT, AccumT> Optional<Iterable<WindowedValue<AccumT>>> combine
 
   /**
    * An implementation of
-   * {@link org.apache.beam.sdk.util.Reshuffle} for the Spark runner.
+   * {@link Reshuffle} for the Spark runner.
    */
   public static <K, V> JavaRDD<WindowedValue<KV<K, V>>> reshuffle(
       JavaRDD<WindowedValue<KV<K, V>>> rdd,
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index 77d2c0e693e6..8a8e24677759 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -50,12 +50,12 @@
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.CombineFnUtil;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index 08f0e172b638..2c4a7476d0ea 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -66,13 +66,13 @@
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.CombineFnUtil;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index a3af8d9860e9..2e11177b3184 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -33,7 +33,6 @@
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
 import com.google.common.collect.TreeMultimap;
-
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.channels.ReadableByteChannel;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReifyTimestamps.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ReifyTimestamps.java
similarity index 92%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReifyTimestamps.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ReifyTimestamps.java
index 3b291af01f21..0b1ab2588672 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReifyTimestamps.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ReifyTimestamps.java
@@ -16,11 +16,8 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.transforms;
 
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
@@ -29,7 +26,7 @@
  * {@link PTransform PTransforms} for reifying the timestamp of values and reemitting the original
  * value with the original timestamp.
  */
-public class ReifyTimestamps {
+class ReifyTimestamps {
   private ReifyTimestamps() {}
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Reshuffle.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
similarity index 79%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/Reshuffle.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
index 887f01103289..5394826e8551 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Reshuffle.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
@@ -15,15 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.transforms;
 
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.ReshuffleTrigger;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.IdentityWindowFn;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
@@ -31,18 +30,22 @@
 import org.joda.time.Duration;
 
 /**
- * A {@link PTransform} that returns a {@link PCollection} equivalent to its input but operationally
- * provides some of the side effects of a {@link GroupByKey}, in particular preventing fusion of
- * the surrounding transforms, checkpointing and deduplication by id (see
- * {@link ValueWithRecordId}).
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
  *
- * <p>Performs a {@link GroupByKey} so that the data is key-partitioned. Configures the
- * {@link WindowingStrategy} so that no data is dropped, but doesn't affect the need for
- * the user to specify allowed lateness and accumulation mode before a user-inserted GroupByKey.
+ * <p>A {@link PTransform} that returns a {@link PCollection} equivalent to its input but
+ * operationally provides some of the side effects of a {@link GroupByKey}, in particular preventing
+ * fusion of the surrounding transforms, checkpointing and deduplication by id.
+ *
+ * <p>Performs a {@link GroupByKey} so that the data is key-partitioned. Configures the {@link
+ * WindowingStrategy} so that no data is dropped, but doesn't affect the need for the user to
+ * specify allowed lateness and accumulation mode before a user-inserted GroupByKey.
  *
  * @param <K> The type of key being reshuffled on.
  * @param <V> The type of value being reshuffled.
+ * @deprecated this transform's intended side effects are not portable; it will likely be removed
  */
+@Internal
+@Deprecated
 public class Reshuffle<K, V> extends PTransform<PCollection<KV<K, V>>, PCollection<KV<K, V>>> {
 
   private Reshuffle() {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReshuffleTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/ReshuffleTrigger.java
similarity index 75%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReshuffleTrigger.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/ReshuffleTrigger.java
index 8dd648adcd50..ceb7011a5254 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReshuffleTrigger.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/ReshuffleTrigger.java
@@ -15,19 +15,25 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.transforms.windowing;
 
 import java.util.List;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.joda.time.Instant;
 
 /**
- * The trigger used with {@link Reshuffle} which triggers on every element
- * and never buffers state.
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
+ *
+ * <p>The trigger used with {@link Reshuffle} which triggers on every element and never buffers
+ * state.
  *
  * @param <W> The kind of window that is being reshuffled.
+ * @deprecated The intended side effect of {@link Reshuffle} is not portable; it will likely be
+ *     removed
  */
+@Internal
+@Deprecated
 public class ReshuffleTrigger<W extends BoundedWindow> extends Trigger {
 
   public ReshuffleTrigger() {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
index 50e9c1d4732e..9cb642a77ff4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
@@ -54,7 +54,6 @@
 import org.apache.beam.sdk.transforms.windowing.Sessions;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReifyTimestampsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReifyTimestampsTest.java
similarity index 93%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReifyTimestampsTest.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReifyTimestampsTest.java
index 2942efdf38c7..181433eaafc1 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReifyTimestampsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReifyTimestampsTest.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.transforms;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertThat;
@@ -25,11 +25,6 @@
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.ValidatesRunner;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.WithTimestamps;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
similarity index 97%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
index 71bfdb5c7585..1038fd63810b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.transforms;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertEquals;
@@ -30,11 +30,6 @@
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.ValidatesRunner;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.Values;
-import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTriggerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/ReshuffleTriggerTest.java
similarity index 90%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTriggerTest.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/ReshuffleTriggerTest.java
index 63c71ed0b72c..5985ecbc209e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ReshuffleTriggerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/ReshuffleTriggerTest.java
@@ -15,12 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.transforms.windowing;
 
 import static org.junit.Assert.assertEquals;
 
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.Trigger;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
diff --git a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
index 689dbff2fb30..e98bfa74a23c 100644
--- a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
+++ b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
@@ -26,7 +26,6 @@
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
-
 import org.apache.beam.sdk.extensions.sorter.SorterTestUtils.SorterGenerator;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
diff --git a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java
index d534c8765c85..0a7251711c9c 100644
--- a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java
+++ b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java
@@ -17,14 +17,12 @@
 import com.google.common.collect.Lists;
 import com.google.common.hash.HashCode;
 import com.google.common.hash.Hashing;
-
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
-
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
index f4221354621b..ba64ab108656 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
@@ -40,12 +40,12 @@
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 318ea8910b8d..09508e0a4628 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -27,7 +27,6 @@
 import com.google.cloud.hadoop.util.ApiErrorExtractor;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.hash.Hashing;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -35,7 +34,6 @@
 import java.util.UUID;
 import java.util.regex.Matcher;
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResolveOptions;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
index 710c934a70b6..aee88e520857 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
@@ -33,7 +33,6 @@
 import java.io.ObjectInputStream;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
-
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java
index db172dc0b12e..1ac216f87e0f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CalculateSchemas.java
@@ -25,7 +25,6 @@
 import java.util.Map;
 import org.apache.beam.sdk.transforms.Distinct;
 import org.apache.beam.sdk.transforms.DoFn;
-
 import org.apache.beam.sdk.transforms.Keys;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
index 210a072509a0..3dc10b08a024 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
@@ -29,7 +29,6 @@
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.transforms.DoFn;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
index 20b47e10f9a0..886236b5f00d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
@@ -22,10 +22,10 @@
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
index 962e2cde7f4d..7a82c54e277a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
@@ -19,7 +19,6 @@
 package org.apache.beam.sdk.io.gcp.bigquery;
 
 import com.google.api.services.bigquery.model.TableReference;
-
 import java.io.Serializable;
 import java.util.Objects;
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
index 4136fa06084a..66004b22b6a5 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
@@ -23,7 +23,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.WriteBundlesToFiles.Result;
 import org.apache.beam.sdk.transforms.DoFn;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
index f575a3da1b77..bf9d9f1f8000 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
@@ -28,7 +28,6 @@
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
index e7dba2a87a96..83ff16bb67d1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
@@ -31,7 +31,6 @@
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.MoveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
index 367aeb78ae10..18ff68889cca 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeBigQueryServices.java
@@ -24,13 +24,11 @@
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.common.collect.Lists;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.NoSuchElementException;
-
 import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.ListCoder;
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
index ee3af0ba9892..50be0bb8156d 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
@@ -42,7 +42,6 @@
 import com.google.common.collect.HashBasedTable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
-
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.FileReader;
@@ -54,7 +53,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
-
 import org.apache.avro.Schema;
 import org.apache.avro.file.DataFileWriter;
 import org.apache.avro.generic.GenericDatumWriter;
@@ -69,7 +67,6 @@
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
 import org.apache.beam.sdk.util.FluentBackoff;
-
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.Transport;
 import org.joda.time.Duration;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
index d52723b79bc0..89150698114c 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableContainer.java
@@ -19,7 +19,6 @@
 
 import com.google.api.services.bigquery.model.Table;
 import com.google.api.services.bigquery.model.TableRow;
-
 import java.util.ArrayList;
 import java.util.List;
 
diff --git a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ConfigurableEmployeeInputFormat.java b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ConfigurableEmployeeInputFormat.java
index 40f949baf7cf..1d3190eb130d 100644
--- a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ConfigurableEmployeeInputFormat.java
+++ b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ConfigurableEmployeeInputFormat.java
@@ -19,7 +19,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
diff --git a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/EmployeeInputFormat.java b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/EmployeeInputFormat.java
index 206f9ab49f51..a641707c2fcf 100644
--- a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/EmployeeInputFormat.java
+++ b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/EmployeeInputFormat.java
@@ -19,7 +19,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-
 import org.apache.beam.sdk.values.KV;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
diff --git a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ReuseObjectsEmployeeInputFormat.java b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ReuseObjectsEmployeeInputFormat.java
index fbe74ec05ada..defef472d5a9 100644
--- a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ReuseObjectsEmployeeInputFormat.java
+++ b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/ReuseObjectsEmployeeInputFormat.java
@@ -19,7 +19,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-
 import org.apache.beam.sdk.values.KV;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
diff --git a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/TestEmployeeDataSet.java b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/TestEmployeeDataSet.java
index 4a8fe951c347..4dd7004f2dfc 100644
--- a/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/TestEmployeeDataSet.java
+++ b/sdks/java/io/hadoop/input-format/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/TestEmployeeDataSet.java
@@ -16,10 +16,8 @@
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
-
 import java.util.ArrayList;
 import java.util.List;
-
 import org.apache.beam.sdk.values.KV;
 import org.apache.hadoop.io.Text;
 /**
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java
index 51cbd5a2d124..99d371df4f55 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java
+++ b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithElasticTest.java
@@ -24,7 +24,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-
 import org.apache.beam.sdk.io.common.HashingFn;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithEmbeddedCassandraTest.java b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithEmbeddedCassandraTest.java
index 50b2c4f09947..fd3590c7808d 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithEmbeddedCassandraTest.java
+++ b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/HIFIOWithEmbeddedCassandraTest.java
@@ -23,9 +23,7 @@
 import com.datastax.driver.core.SocketOptions;
 import com.datastax.driver.mapping.annotations.Column;
 import com.datastax.driver.mapping.annotations.Table;
-
 import java.io.Serializable;
-
 import org.apache.beam.sdk.io.common.HashingFn;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOCassandraIT.java b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOCassandraIT.java
index ab8203be0900..5a3caed2e9f5 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOCassandraIT.java
+++ b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOCassandraIT.java
@@ -18,9 +18,7 @@
 package org.apache.beam.sdk.io.hadoop.inputformat.integration.tests;
 
 import com.datastax.driver.core.Row;
-
 import java.io.Serializable;
-
 import org.apache.beam.sdk.io.common.HashingFn;
 import org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIO;
 import org.apache.beam.sdk.io.hadoop.inputformat.custom.options.HIFTestOptions;
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOElasticIT.java b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOElasticIT.java
index 08c0668e243a..f7f496debadf 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOElasticIT.java
+++ b/sdks/java/io/hadoop/jdk1.8-tests/src/test/java/org/apache/beam/sdk/io/hadoop/inputformat/integration/tests/HIFIOElasticIT.java
@@ -16,7 +16,6 @@
 
 import java.io.IOException;
 import java.io.Serializable;
-
 import org.apache.beam.sdk.io.common.HashingFn;
 import org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIO;
 import org.apache.beam.sdk.io.hadoop.inputformat.custom.options.HIFTestOptions;
diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
index 2eb53dd6361a..8092da6b971a 100644
--- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
+++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java
@@ -27,10 +27,8 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Random;
-
 import javax.annotation.Nullable;
 import javax.sql.DataSource;
-
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
index 3eaa5b77f6be..e8ffad6c56bb 100644
--- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
+++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
@@ -24,7 +24,6 @@
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.List;
-
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsRecord.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsRecord.java
index 15a1109df3c6..1c3b3b380d97 100644
--- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsRecord.java
+++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsRecord.java
@@ -20,7 +20,6 @@
 import java.io.Serializable;
 import java.util.Map;
 import java.util.Objects;
-
 import javax.annotation.Nullable;
 import javax.jms.Destination;
 
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java
index 61a382d6ee2a..64dbad450d51 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java
@@ -18,11 +18,9 @@
 package org.apache.beam.sdk.io.kafka;
 
 import com.google.common.base.Joiner;
-
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.List;
-
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.DefaultCoder;
 import org.apache.beam.sdk.io.UnboundedSource;
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java
index e0e400eb5a8e..235fb1f8b2ae 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java
@@ -19,7 +19,6 @@
 
 import java.io.Serializable;
 import java.util.Arrays;
-
 import org.apache.beam.sdk.values.KV;
 
 /**
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java
index ca552fb054a4..a1655860b36a 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java
@@ -21,7 +21,6 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.util.Map;
-
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.util.CoderUtils;
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java
index 1044d6fe1792..84b617ee8ac4 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java
@@ -22,7 +22,6 @@
 
 import java.util.Map;
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.util.CoderUtils;
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantDeserializer.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantDeserializer.java
index fe4749f9f82b..76ceed56d99e 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantDeserializer.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantDeserializer.java
@@ -19,7 +19,6 @@
 package org.apache.beam.sdk.io.kafka.serialization;
 
 import java.util.Map;
-
 import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.LongDeserializer;
 import org.joda.time.Instant;
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantSerializer.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantSerializer.java
index 8fa4429d373b..0d69b2033f28 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantSerializer.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/InstantSerializer.java
@@ -19,7 +19,6 @@
 package org.apache.beam.sdk.io.kafka.serialization;
 
 import java.util.Map;
-
 import org.apache.kafka.common.serialization.LongSerializer;
 import org.apache.kafka.common.serialization.Serializer;
 import org.joda.time.Instant;
diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
index 0868ed443c84..b63775da7827 100644
--- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
+++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
@@ -30,7 +30,6 @@
 import com.mongodb.gridfs.GridFSDBFile;
 import com.mongodb.gridfs.GridFSInputFile;
 import com.mongodb.util.JSON;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -40,9 +39,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
-
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.SerializableCoder;
diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
index 8e7f03b0973f..826af1cc59a7 100644
--- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
+++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
@@ -25,7 +25,6 @@
 import com.mongodb.gridfs.GridFS;
 import com.mongodb.gridfs.GridFSDBFile;
 import com.mongodb.gridfs.GridFSInputFile;
-
 import de.flapdoodle.embed.mongo.MongodExecutable;
 import de.flapdoodle.embed.mongo.MongodProcess;
 import de.flapdoodle.embed.mongo.MongodStarter;
@@ -37,7 +36,6 @@
 import de.flapdoodle.embed.mongo.distribution.Version;
 import de.flapdoodle.embed.process.io.file.Files;
 import de.flapdoodle.embed.process.runtime.Network;
-
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -54,7 +52,6 @@
 import java.util.List;
 import java.util.Random;
 import java.util.Scanner;
-
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
index c36b7c8d344f..454c6ba1ea7b 100644
--- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
+++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
@@ -23,7 +23,6 @@
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.MongoDatabase;
-
 import de.flapdoodle.embed.mongo.MongodExecutable;
 import de.flapdoodle.embed.mongo.MongodProcess;
 import de.flapdoodle.embed.mongo.MongodStarter;
@@ -35,12 +34,10 @@
 import de.flapdoodle.embed.mongo.distribution.Version;
 import de.flapdoodle.embed.process.io.file.Files;
 import de.flapdoodle.embed.process.runtime.Network;
-
 import java.io.File;
 import java.io.Serializable;
 import java.net.ServerSocket;
 import java.util.ArrayList;
-
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Count;
@@ -50,7 +47,6 @@
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
-
 import org.bson.Document;
 import org.junit.After;
 import org.junit.Assert;
diff --git a/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java b/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java
index 0f25b0fb8357..228a85d77ab2 100644
--- a/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java
+++ b/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java
@@ -20,7 +20,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.auto.value.AutoValue;
-
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -28,9 +27,7 @@
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.UUID;
-
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;

From b2553caf1350eaea3caefe55d5af414694c96424 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 20:19:51 -0700
Subject: [PATCH 058/387] Move ValueWithRecordId to sdk.values, annotated

---
 .../org/apache/beam/runners/dataflow/DataflowRunner.java  | 2 +-
 .../org/apache/beam/runners/spark/TestSparkRunner.java    | 2 +-
 .../beam/sdk/io/BoundedReadFromUnboundedSource.java       | 2 +-
 .../beam/sdk/{util => values}/ValueWithRecordId.java      | 8 ++++++--
 .../beam/sdk/{util => values}/ValueWithRecordIdTest.java  | 4 ++--
 5 files changed, 11 insertions(+), 7 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => values}/ValueWithRecordId.java (93%)
 rename sdks/java/core/src/test/java/org/apache/beam/sdk/{util => values}/ValueWithRecordIdTest.java (92%)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 7123316f436e..57da61b905b7 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -121,7 +121,6 @@
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.util.ReleaseInfo;
-import org.apache.beam.sdk.util.ValueWithRecordId;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
@@ -132,6 +131,7 @@
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.ValueWithRecordId;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.DateTimeUtils;
 import org.joda.time.DateTimeZone;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 6808d7b6b132..a6851c49ded5 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -48,11 +48,11 @@
 import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.util.ValueWithRecordId;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.ValueWithRecordId;
 import org.apache.commons.io.FileUtils;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
index e54176fec0f5..d9adf9297db5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
@@ -37,9 +37,9 @@
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.NameUtils;
-import org.apache.beam.sdk.util.ValueWithRecordId;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.ValueWithRecordId;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ValueWithRecordId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
similarity index 93%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/ValueWithRecordId.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
index 9902aa7a1fb4..0d92f4075d55 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ValueWithRecordId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.values;
 
 import com.google.common.base.MoreObjects;
 import java.io.IOException;
@@ -24,16 +24,20 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.transforms.DoFn;
 
 /**
- * Immutable struct containing a value as well as a unique id identifying the value.
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
+ *
+ * <p>Immutable struct containing a value as well as a unique id identifying the value.
  *
  * @param <ValueT> the underlying value type
  */
+@Internal
 public class ValueWithRecordId<ValueT> {
   private final ValueT value;
   private final byte[] id;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ValueWithRecordIdTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/ValueWithRecordIdTest.java
similarity index 92%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/ValueWithRecordIdTest.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/values/ValueWithRecordIdTest.java
index e3a2dc6df80b..987c9af3baff 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ValueWithRecordIdTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/ValueWithRecordIdTest.java
@@ -15,11 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.values;
 
 import org.apache.beam.sdk.testing.CoderProperties;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.ValueWithRecordId.ValueWithRecordIdCoder;
+import org.apache.beam.sdk.values.ValueWithRecordId.ValueWithRecordIdCoder;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;

From c83cc744a69f735ac134471705e3403b9d5edd34 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 3 May 2017 20:34:54 -0700
Subject: [PATCH 059/387] Move some PCollectionView bits out of util

---
 .../org/apache/beam/runners/apex/ApexRunner.java |  2 +-
 .../construction/PTransformMatchersTest.java     |  2 +-
 .../runners/direct}/PCollectionViewWindow.java   |  2 +-
 .../beam/runners/direct/SideInputContainer.java  |  1 -
 .../runners/direct/SideInputContainerTest.java   |  2 +-
 .../runners/direct/ViewEvaluatorFactoryTest.java |  2 +-
 .../runners/direct/ViewOverrideFactoryTest.java  |  2 +-
 .../direct/WriteWithShardingFactoryTest.java     |  2 +-
 .../flink/FlinkStreamingViewOverrides.java       |  2 +-
 .../runners/dataflow/BatchViewOverrides.java     |  2 +-
 .../org/apache/beam/sdk/transforms/Combine.java  |  2 +-
 .../org/apache/beam/sdk/transforms/View.java     |  2 +-
 .../sdk/{util => values}/PCollectionViews.java   | 16 +++++++---------
 .../beam/sdk/transforms/DoFnTesterTest.java      |  2 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java |  2 +-
 15 files changed, 20 insertions(+), 23 deletions(-)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => runners/direct-java/src/main/java/org/apache/beam/runners/direct}/PCollectionViewWindow.java (98%)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{util => values}/PCollectionViews.java (97%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index 1c845c6c628b..e1828c3217c6 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -60,9 +60,9 @@
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.View.AsIterable;
 import org.apache.beam.sdk.transforms.View.AsSingleton;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.hadoop.conf.Configuration;
 
 /**
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index cb28c3449295..6271234db021 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -60,13 +60,13 @@
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PCollectionList;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViewWindow.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PCollectionViewWindow.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViewWindow.java
rename to runners/direct-java/src/main/java/org/apache/beam/runners/direct/PCollectionViewWindow.java
index 410c8cea6832..7a7d8ff238d7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViewWindow.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PCollectionViewWindow.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.runners.direct;
 
 import java.util.Objects;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
index 380dc65b0090..43da92ff916d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
@@ -39,7 +39,6 @@
 import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.PCollectionViewWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.WindowingStrategy;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
index d4ca9fdd76b7..5e7c79918514 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
@@ -45,11 +45,11 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
index 9560e94e0249..d8869b2ca25d 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
@@ -34,10 +34,10 @@
 import org.apache.beam.sdk.transforms.Values;
 import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
 import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.joda.time.Instant;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
index a36787af86fc..eda00a785e0c 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
@@ -39,9 +39,9 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
 import org.junit.Rule;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
index 6fffd1a479ec..a2b0c5ce7733 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
@@ -54,9 +54,9 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFnTester;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
index f955f2a573ff..ce1c89555520 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
@@ -30,10 +30,10 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 
 /**
  * Flink streaming overrides for various view (side input) transforms.
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index 1ff8a3f3affd..debaf59ef93e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -74,7 +74,6 @@
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
@@ -84,6 +83,7 @@
 import org.apache.beam.sdk.values.PCollectionList;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.WindowingStrategy;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 1be948f779fb..666db3b80a65 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -57,13 +57,13 @@
 import org.apache.beam.sdk.util.AppliedCombineFn;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.util.NameUtils.NameOverride;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
index d17d423441a9..d7b81455b63f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
@@ -25,10 +25,10 @@
 import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 
 /**
  * Transforms for creating {@link PCollectionView PCollectionViews} from
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
index a07bc5e5818b..74887c71b368 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.values;
 
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
@@ -33,6 +33,7 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.transforms.Materialization;
@@ -41,18 +42,15 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PValueBase;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.util.WindowedValue;
 
 /**
- * Implementations of {@link PCollectionView} shared across the SDK.
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
  *
- * <p>For internal use only, subject to change.
+ * <p>Implementations of {@link PCollectionView} shared across the SDK.
  */
+@Internal
 public class PCollectionViews {
 
   /**
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
index f74d67344473..1bb71bbf1a18 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
@@ -34,10 +34,10 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.hamcrest.Matchers;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 38da2d94c112..a3b21eebd802 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -121,11 +121,11 @@
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
 import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.sdk.util.MimeTypes;
-import org.apache.beam.sdk.util.PCollectionViews;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.sdk.values.ValueInSingleWindow;

From 02e80a7d5890ac2281d970cb1977399f79431d5b Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 4 May 2017 16:09:26 -0700
Subject: [PATCH 060/387] Update Dataflow worker to 20170504-2

---
 runners/google-cloud-dataflow-java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 0367bd8a95a1..e21b6de0f784 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170504</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170504-2</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>

From a5faa413d58be7e0840f9aefbf746d9fdb4c2830 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 4 May 2017 15:26:07 -0700
Subject: [PATCH 061/387] [BEAM-2144] Remove transform.reflect and utils from
 Javadocs

---
 sdks/java/javadoc/ant.xml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sdks/java/javadoc/ant.xml b/sdks/java/javadoc/ant.xml
index 80dbdc2fe090..288f48bf9baa 100644
--- a/sdks/java/javadoc/ant.xml
+++ b/sdks/java/javadoc/ant.xml
@@ -79,7 +79,7 @@
         offline="true" packageListLoc="joda-docs"/>
       <link href="http://junit.sourceforge.net/javadoc/"
             offline="true" packageListLoc="junit-docs"/>
-      
+
       <excludepackage name="org.apache.beam.examples.*"/>
       <excludepackage name="org.apache.beam.fn.*"/>
       <excludepackage name="org.apache.beam.runners.apex.translation.*"/>
@@ -89,6 +89,8 @@
       <excludepackage name="org.apache.beam.runners.flink.translation.*"/>
       <excludepackage name="org.apache.beam.runners.spark.examples.*"/>
       <excludepackage name="org.apache.beam.runners.spark.translation.*"/>
+      <excludepackage name="org.apache.beam.sdk.transforms.reflect.*"/>
+      <excludepackage name="org.apache.beam.sdk.util.*"/>
     </javadoc>
     <!-- make a jar file to push to central -->
     <jar destfile="${output.jar}" basedir="${output.dir}"/>

From 040dbd4fd21a31dd3eb0f34fb9610ce857111db8 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 4 May 2017 17:01:24 -0700
Subject: [PATCH 062/387] Update expectation in ProcessBundleHandlerTest

---
 .../beam/fn/harness/control/ProcessBundleHandlerTest.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 3e06369320af..748ffea1f13c 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.beam.fn.harness.control;
 
+import static org.apache.beam.sdk.util.WindowedValue.timestampedValueInGlobalWindow;
 import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
@@ -433,7 +434,10 @@ public void testCreatingAndProcessingDoFn() throws Exception {
     additionalOutputValues.clear();
 
     Iterables.getOnlyElement(finishFunctions).run();
-    assertThat(mainOutputValues, contains(valueInGlobalWindow("FinishBundle")));
+    assertThat(
+        mainOutputValues,
+        contains(
+            timestampedValueInGlobalWindow("FinishBundle", GlobalWindow.INSTANCE.maxTimestamp())));
     mainOutputValues.clear();
   }
 

From c158e05ccdd1d06bd4f49c972288bf28c4e5d703 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Thu, 4 May 2017 16:14:34 -0700
Subject: [PATCH 063/387] [BEAM-1871] Move out test only utilities used only in
 examples/java out of sdks/java/core.

---
 examples/java/pom.xml                                | 12 ++++++++++++
 .../apache/beam/examples/WindowedWordCountIT.java    |  8 ++++----
 .../java/org/apache/beam/examples/WordCountIT.java   |  2 +-
 .../beam/examples/testing}/ExplicitShardedFile.java  |  3 ++-
 .../beam/examples}/testing/FileChecksumMatcher.java  |  5 ++---
 .../examples}/testing/FileChecksumMatcherTest.java   |  3 ++-
 .../beam/examples/testing}/NumberedShardedFile.java  |  3 ++-
 .../examples/testing}/NumberedShardedFileTest.java   |  2 +-
 .../apache/beam/examples/testing}/ShardedFile.java   |  2 +-
 .../apache/beam/sdk/coders/StructuralByteArray.java  |  5 ++---
 .../apache/beam/sdk/testing/MatcherDeserializer.java |  4 ++--
 .../apache/beam/sdk/testing/MatcherSerializer.java   |  4 ++--
 .../java/org/apache/beam/sdk/util/CoderUtils.java    |  8 +++++---
 13 files changed, 38 insertions(+), 23 deletions(-)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => examples/java/src/test/java/org/apache/beam/examples/testing}/ExplicitShardedFile.java (98%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => examples/java/src/test/java/org/apache/beam/examples}/testing/FileChecksumMatcher.java (97%)
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => examples/java/src/test/java/org/apache/beam/examples}/testing/FileChecksumMatcherTest.java (98%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => examples/java/src/test/java/org/apache/beam/examples/testing}/NumberedShardedFile.java (98%)
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk/util => examples/java/src/test/java/org/apache/beam/examples/testing}/NumberedShardedFileTest.java (99%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => examples/java/src/test/java/org/apache/beam/examples/testing}/ShardedFile.java (97%)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index d673da27fe29..09473cd48827 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -491,6 +491,11 @@
       <artifactId>google-auth-library-credentials</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
@@ -552,6 +557,13 @@
       For testing the example itself, use the direct runner. This is separate from
       the use of ValidatesRunner tests for testing a particular runner.
     -->
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-core</artifactId>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-runners-direct-java</artifactId>
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index b5eddb5d17e7..01bc4027f814 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -33,21 +33,21 @@
 
 import org.apache.beam.examples.common.ExampleUtils;
 import org.apache.beam.examples.common.WriteOneFilePerWindow.PerWindowFiles;
+import org.apache.beam.examples.testing.ExplicitShardedFile;
+import org.apache.beam.examples.testing.FileChecksumMatcher;
+import org.apache.beam.examples.testing.NumberedShardedFile;
+import org.apache.beam.examples.testing.ShardedFile;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.StreamingOptions;
-import org.apache.beam.sdk.testing.FileChecksumMatcher;
 import org.apache.beam.sdk.testing.SerializableMatcher;
 import org.apache.beam.sdk.testing.StreamingIT;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.util.ExplicitShardedFile;
 import org.apache.beam.sdk.util.FluentBackoff;
-import org.apache.beam.sdk.util.NumberedShardedFile;
-import org.apache.beam.sdk.util.ShardedFile;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 import org.joda.time.Duration;
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
index 1660b61d1672..236ca9cb0780 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
@@ -20,10 +20,10 @@
 
 import java.util.Date;
 import org.apache.beam.examples.WordCount.WordCountOptions;
+import org.apache.beam.examples.testing.FileChecksumMatcher;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.FileChecksumMatcher;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.junit.BeforeClass;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java b/examples/java/src/test/java/org/apache/beam/examples/testing/ExplicitShardedFile.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
rename to examples/java/src/test/java/org/apache/beam/examples/testing/ExplicitShardedFile.java
index 0f184de7f8f4..1dc7a62d07f2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/testing/ExplicitShardedFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.util;
+package org.apache.beam.examples.testing;
 
 import com.google.api.client.util.BackOff;
 import com.google.api.client.util.BackOffUtils;
@@ -35,6 +35,7 @@
 import java.util.List;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java b/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcher.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
rename to examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcher.java
index 82a6b7117636..8a0af1199f1e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcher.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.testing;
+package org.apache.beam.examples.testing;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -31,9 +31,8 @@
 import java.util.regex.Pattern;
 import javax.annotation.Nonnull;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.testing.SerializableMatcher;
 import org.apache.beam.sdk.util.FluentBackoff;
-import org.apache.beam.sdk.util.NumberedShardedFile;
-import org.apache.beam.sdk.util.ShardedFile;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 import org.joda.time.Duration;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java b/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcherTest.java
similarity index 98%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
rename to examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcherTest.java
index 4ee67501817c..4d6eb6bbef49 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcherTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.testing;
+package org.apache.beam.examples.testing;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
@@ -27,6 +27,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Pattern;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java b/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFile.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
rename to examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFile.java
index e18dd96f0046..f0b9c2d6045f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.util;
+package org.apache.beam.examples.testing;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -44,6 +44,7 @@
 import javax.annotation.Nonnull;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java b/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFileTest.java
similarity index 99%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
rename to examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFileTest.java
index 43a91665c9c3..83b8a4f136d4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFileTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.examples.testing;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java b/examples/java/src/test/java/org/apache/beam/examples/testing/ShardedFile.java
similarity index 97%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
rename to examples/java/src/test/java/org/apache/beam/examples/testing/ShardedFile.java
index ec9ed641976c..cd9537cb326d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/testing/ShardedFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.util;
+package org.apache.beam.examples.testing;
 
 import com.google.api.client.util.BackOff;
 import com.google.api.client.util.Sleeper;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
index 226f79cd429d..0ab0dea9bdab 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
@@ -17,8 +17,7 @@
  */
 package org.apache.beam.sdk.coders;
 
-import static com.google.api.client.util.Base64.encodeBase64String;
-
+import com.google.common.io.BaseEncoding;
 import java.util.Arrays;
 
 /**
@@ -53,6 +52,6 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return "base64:" + encodeBase64String(value);
+    return "base64:" + BaseEncoding.base64().encode(value);
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
index 6ca07badac92..e7aa5a7f4283 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
@@ -22,7 +22,7 @@
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonDeserializer;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.client.util.Base64;
+import com.google.common.io.BaseEncoding;
 import java.io.IOException;
 import org.apache.beam.sdk.util.SerializableUtils;
 
@@ -36,7 +36,7 @@ public SerializableMatcher<?> deserialize(JsonParser jsonParser,
       throws IOException, JsonProcessingException {
     ObjectNode node = jsonParser.readValueAsTree();
     String matcher = node.get("matcher").asText();
-    byte[] in = Base64.decodeBase64(matcher);
+    byte[] in = BaseEncoding.base64().decode(matcher);
     return (SerializableMatcher<?>) SerializableUtils
         .deserializeFromByteArray(in, "SerializableMatcher");
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
index 2b4584c8cdac..35375f66047d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
@@ -21,7 +21,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.SerializerProvider;
-import com.google.api.client.util.Base64;
+import com.google.common.io.BaseEncoding;
 import java.io.IOException;
 import org.apache.beam.sdk.util.SerializableUtils;
 
@@ -33,7 +33,7 @@ class MatcherSerializer extends JsonSerializer<SerializableMatcher<?>> {
   public void serialize(SerializableMatcher<?> matcher, JsonGenerator jsonGenerator,
       SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
     byte[] out = SerializableUtils.serializeToByteArray(matcher);
-    String encodedString = Base64.encodeBase64String(out);
+    String encodedString = BaseEncoding.base64().encode(out);
     jsonGenerator.writeStartObject();
     jsonGenerator.writeStringField("matcher", encodedString);
     jsonGenerator.writeEndObject();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
index 3380a102e68a..e3ae485f5d18 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
@@ -17,8 +17,8 @@
  */
 package org.apache.beam.sdk.util;
 
-import com.google.api.client.util.Base64;
 import com.google.common.base.Throwables;
+import com.google.common.io.BaseEncoding;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -163,7 +163,7 @@ public static <T> T clone(Coder<T> coder, T value) throws CoderException {
   public static <T> String encodeToBase64(Coder<T> coder, T value)
       throws CoderException {
     byte[] rawValue = encodeToByteArray(coder, value);
-    return Base64.encodeBase64URLSafeString(rawValue);
+    return BaseEncoding.base64Url().omitPadding().encode(rawValue);
   }
 
   /**
@@ -171,7 +171,9 @@ public static <T> String encodeToBase64(Coder<T> coder, T value)
    */
   public static <T> T decodeFromBase64(Coder<T> coder, String encodedValue) throws CoderException {
     return decodeFromSafeStream(
-        coder, new ByteArrayInputStream(Base64.decodeBase64(encodedValue)), Coder.Context.OUTER);
+        coder,
+        new ByteArrayInputStream(BaseEncoding.base64Url().omitPadding().decode(encodedValue)),
+        Coder.Context.OUTER);
   }
 
   /**

From b130d7aac466860ed5a7abec0bef33d9e0dd3c6d Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 4 May 2017 18:14:13 -0700
Subject: [PATCH 064/387] Revert "This closes #2905"

I misread Jenkins and should not have merged. I am sorry.

This reverts commit 9f27c33ec7e7c61afbca0395f932275b354eb428, reversing
changes made to 5fc3d335919207c23bc6fd2047e9e38351754ff1.
---
 examples/java/pom.xml                                | 12 ------------
 .../apache/beam/examples/WindowedWordCountIT.java    |  8 ++++----
 .../java/org/apache/beam/examples/WordCountIT.java   |  2 +-
 .../apache/beam/sdk/coders/StructuralByteArray.java  |  5 +++--
 .../beam/sdk}/testing/FileChecksumMatcher.java       |  5 +++--
 .../apache/beam/sdk/testing/MatcherDeserializer.java |  4 ++--
 .../apache/beam/sdk/testing/MatcherSerializer.java   |  4 ++--
 .../java/org/apache/beam/sdk/util/CoderUtils.java    |  8 +++-----
 .../apache/beam/sdk/util}/ExplicitShardedFile.java   |  3 +--
 .../apache/beam/sdk/util}/NumberedShardedFile.java   |  3 +--
 .../java/org/apache/beam/sdk/util}/ShardedFile.java  |  2 +-
 .../beam/sdk}/testing/FileChecksumMatcherTest.java   |  3 +--
 .../beam/sdk/util}/NumberedShardedFileTest.java      |  2 +-
 13 files changed, 23 insertions(+), 38 deletions(-)
 rename {examples/java/src/test/java/org/apache/beam/examples => sdks/java/core/src/main/java/org/apache/beam/sdk}/testing/FileChecksumMatcher.java (97%)
 rename {examples/java/src/test/java/org/apache/beam/examples/testing => sdks/java/core/src/main/java/org/apache/beam/sdk/util}/ExplicitShardedFile.java (98%)
 rename {examples/java/src/test/java/org/apache/beam/examples/testing => sdks/java/core/src/main/java/org/apache/beam/sdk/util}/NumberedShardedFile.java (98%)
 rename {examples/java/src/test/java/org/apache/beam/examples/testing => sdks/java/core/src/main/java/org/apache/beam/sdk/util}/ShardedFile.java (97%)
 rename {examples/java/src/test/java/org/apache/beam/examples => sdks/java/core/src/test/java/org/apache/beam/sdk}/testing/FileChecksumMatcherTest.java (98%)
 rename {examples/java/src/test/java/org/apache/beam/examples/testing => sdks/java/core/src/test/java/org/apache/beam/sdk/util}/NumberedShardedFileTest.java (99%)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 09473cd48827..d673da27fe29 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -491,11 +491,6 @@
       <artifactId>google-auth-library-credentials</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.code.findbugs</groupId>
-      <artifactId>jsr305</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
@@ -557,13 +552,6 @@
       For testing the example itself, use the direct runner. This is separate from
       the use of ValidatesRunner tests for testing a particular runner.
     -->
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-java-core</artifactId>
-      <classifier>tests</classifier>
-      <scope>test</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-runners-direct-java</artifactId>
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index 01bc4027f814..b5eddb5d17e7 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -33,21 +33,21 @@
 
 import org.apache.beam.examples.common.ExampleUtils;
 import org.apache.beam.examples.common.WriteOneFilePerWindow.PerWindowFiles;
-import org.apache.beam.examples.testing.ExplicitShardedFile;
-import org.apache.beam.examples.testing.FileChecksumMatcher;
-import org.apache.beam.examples.testing.NumberedShardedFile;
-import org.apache.beam.examples.testing.ShardedFile;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.StreamingOptions;
+import org.apache.beam.sdk.testing.FileChecksumMatcher;
 import org.apache.beam.sdk.testing.SerializableMatcher;
 import org.apache.beam.sdk.testing.StreamingIT;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.util.ExplicitShardedFile;
 import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.NumberedShardedFile;
+import org.apache.beam.sdk.util.ShardedFile;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 import org.joda.time.Duration;
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
index 236ca9cb0780..1660b61d1672 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
@@ -20,10 +20,10 @@
 
 import java.util.Date;
 import org.apache.beam.examples.WordCount.WordCountOptions;
-import org.apache.beam.examples.testing.FileChecksumMatcher;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.FileChecksumMatcher;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.junit.BeforeClass;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
index 0ab0dea9bdab..226f79cd429d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
@@ -17,7 +17,8 @@
  */
 package org.apache.beam.sdk.coders;
 
-import com.google.common.io.BaseEncoding;
+import static com.google.api.client.util.Base64.encodeBase64String;
+
 import java.util.Arrays;
 
 /**
@@ -52,6 +53,6 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return "base64:" + BaseEncoding.base64().encode(value);
+    return "base64:" + encodeBase64String(value);
   }
 }
diff --git a/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
similarity index 97%
rename from examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcher.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
index 8a0af1199f1e..82a6b7117636 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.examples.testing;
+package org.apache.beam.sdk.testing;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -31,8 +31,9 @@
 import java.util.regex.Pattern;
 import javax.annotation.Nonnull;
 import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.testing.SerializableMatcher;
 import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.NumberedShardedFile;
+import org.apache.beam.sdk.util.ShardedFile;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 import org.joda.time.Duration;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
index e7aa5a7f4283..6ca07badac92 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
@@ -22,7 +22,7 @@
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonDeserializer;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.io.BaseEncoding;
+import com.google.api.client.util.Base64;
 import java.io.IOException;
 import org.apache.beam.sdk.util.SerializableUtils;
 
@@ -36,7 +36,7 @@ public SerializableMatcher<?> deserialize(JsonParser jsonParser,
       throws IOException, JsonProcessingException {
     ObjectNode node = jsonParser.readValueAsTree();
     String matcher = node.get("matcher").asText();
-    byte[] in = BaseEncoding.base64().decode(matcher);
+    byte[] in = Base64.decodeBase64(matcher);
     return (SerializableMatcher<?>) SerializableUtils
         .deserializeFromByteArray(in, "SerializableMatcher");
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
index 35375f66047d..2b4584c8cdac 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
@@ -21,7 +21,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.SerializerProvider;
-import com.google.common.io.BaseEncoding;
+import com.google.api.client.util.Base64;
 import java.io.IOException;
 import org.apache.beam.sdk.util.SerializableUtils;
 
@@ -33,7 +33,7 @@ class MatcherSerializer extends JsonSerializer<SerializableMatcher<?>> {
   public void serialize(SerializableMatcher<?> matcher, JsonGenerator jsonGenerator,
       SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
     byte[] out = SerializableUtils.serializeToByteArray(matcher);
-    String encodedString = BaseEncoding.base64().encode(out);
+    String encodedString = Base64.encodeBase64String(out);
     jsonGenerator.writeStartObject();
     jsonGenerator.writeStringField("matcher", encodedString);
     jsonGenerator.writeEndObject();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
index e3ae485f5d18..3380a102e68a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
@@ -17,8 +17,8 @@
  */
 package org.apache.beam.sdk.util;
 
+import com.google.api.client.util.Base64;
 import com.google.common.base.Throwables;
-import com.google.common.io.BaseEncoding;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -163,7 +163,7 @@ public static <T> T clone(Coder<T> coder, T value) throws CoderException {
   public static <T> String encodeToBase64(Coder<T> coder, T value)
       throws CoderException {
     byte[] rawValue = encodeToByteArray(coder, value);
-    return BaseEncoding.base64Url().omitPadding().encode(rawValue);
+    return Base64.encodeBase64URLSafeString(rawValue);
   }
 
   /**
@@ -171,9 +171,7 @@ public static <T> String encodeToBase64(Coder<T> coder, T value)
    */
   public static <T> T decodeFromBase64(Coder<T> coder, String encodedValue) throws CoderException {
     return decodeFromSafeStream(
-        coder,
-        new ByteArrayInputStream(BaseEncoding.base64Url().omitPadding().decode(encodedValue)),
-        Coder.Context.OUTER);
+        coder, new ByteArrayInputStream(Base64.decodeBase64(encodedValue)), Coder.Context.OUTER);
   }
 
   /**
diff --git a/examples/java/src/test/java/org/apache/beam/examples/testing/ExplicitShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
similarity index 98%
rename from examples/java/src/test/java/org/apache/beam/examples/testing/ExplicitShardedFile.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
index 1dc7a62d07f2..0f184de7f8f4 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/testing/ExplicitShardedFile.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.examples.testing;
+package org.apache.beam.sdk.util;
 
 import com.google.api.client.util.BackOff;
 import com.google.api.client.util.BackOffUtils;
@@ -35,7 +35,6 @@
 import java.util.List;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
-import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
similarity index 98%
rename from examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFile.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
index f0b9c2d6045f..e18dd96f0046 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFile.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.examples.testing;
+package org.apache.beam.sdk.util;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -44,7 +44,6 @@
 import javax.annotation.Nonnull;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
-import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/examples/java/src/test/java/org/apache/beam/examples/testing/ShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
similarity index 97%
rename from examples/java/src/test/java/org/apache/beam/examples/testing/ShardedFile.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
index cd9537cb326d..ec9ed641976c 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/testing/ShardedFile.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.examples.testing;
+package org.apache.beam.sdk.util;
 
 import com.google.api.client.util.BackOff;
 import com.google.api.client.util.Sleeper;
diff --git a/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcherTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
similarity index 98%
rename from examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcherTest.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
index 4d6eb6bbef49..4ee67501817c 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/testing/FileChecksumMatcherTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.testing;
+package org.apache.beam.sdk.testing;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
@@ -27,7 +27,6 @@
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Pattern;
 import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
diff --git a/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFileTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
similarity index 99%
rename from examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFileTest.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
index 83b8a4f136d4..43a91665c9c3 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/testing/NumberedShardedFileTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.testing;
+package org.apache.beam.sdk.util;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;

From 6be71c9a1c2ec43478e2ddaedaba67608de129ae Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 4 May 2017 19:03:50 -0700
Subject: [PATCH 065/387] Adjust error message expectation in
 DoFnSignaturesTest

---
 .../beam/sdk/transforms/reflect/DoFnSignaturesTest.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
index 5b97a553c28d..cffb0adf613a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -385,7 +385,7 @@ public void testTimerIdDuplicate() throws Exception {
     thrown.expectMessage("my-id");
     thrown.expectMessage("myfield1");
     thrown.expectMessage("myfield2");
-    thrown.expectMessage(not(mentionsState()));
+    thrown.expectMessage(not(containsString("State"))); // lowercase "state" is in the package name
     thrown.expectMessage(mentionsTimers());
     DoFnSignature sig =
         DoFnSignatures.getSignature(
@@ -407,7 +407,7 @@ public void testTimerIdNonFinal() throws Exception {
     thrown.expectMessage("Timer declarations must be final");
     thrown.expectMessage("Non-final field");
     thrown.expectMessage("myfield");
-    thrown.expectMessage(not(mentionsState()));
+    thrown.expectMessage(not(containsString("State"))); // lowercase "state" is in the package name
     thrown.expectMessage(mentionsTimers());
     DoFnSignature sig =
         DoFnSignatures.getSignature(

From c1811a4adbfb290226d183eac01bbe9bdbce2462 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 4 May 2017 21:34:54 -0700
Subject: [PATCH 066/387] No parallelism for Apex WindowedWordCountIT

---
 examples/java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 65625df297b3..fb2613514e8c 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -305,7 +305,7 @@
                     <include>WordCountIT.java</include>
                     <include>WindowedWordCountIT.java</include>
                   </includes>
-                  <parallel>all</parallel>
+                  <parallel>none</parallel>
                   <threadCount>4</threadCount>
                   <systemPropertyVariables>
                     <beamTestPipelineOptions>

From fc4534cd6e5366a5f12cefebcd52ac1fe7cdde41 Mon Sep 17 00:00:00 2001
From: JingsongLi <lzljs3620320@aliyun.com>
Date: Tue, 4 Apr 2017 18:28:15 +0800
Subject: [PATCH 067/387] [BEAM-1862] SplittableDoFnOperator should close the
 ScheduledExecutorService

---
 .../streaming/SplittableDoFnOperator.java     | 28 ++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
index 7d54cfa51d5d..968fc0a540a7 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
@@ -24,6 +24,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import org.apache.beam.runners.core.ElementAndRestriction;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItems;
@@ -57,6 +59,8 @@ public class SplittableDoFnOperator<
     extends DoFnOperator<
     KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, FnOutputT, OutputT> {
 
+  private transient ScheduledExecutorService executorService;
+
   public SplittableDoFnOperator(
       DoFn<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, FnOutputT> doFn,
       String stepName,
@@ -108,6 +112,8 @@ public TimerInternals timerInternalsForKey(String key) {
       }
     };
 
+    executorService = Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory());
+
     ((SplittableParDo.ProcessFn) doFn).setStateInternalsFactory(stateInternalsFactory);
     ((SplittableParDo.ProcessFn) doFn).setTimerInternalsFactory(timerInternalsFactory);
     ((SplittableParDo.ProcessFn) doFn).setProcessElementInvoker(
@@ -137,7 +143,7 @@ public <AdditionalOutputT> void outputWindowedValue(
               }
             },
             sideInputReader,
-            Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory()),
+            executorService,
             10000,
             Duration.standardSeconds(10)));
   }
@@ -149,4 +155,24 @@ public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) {
             (String) stateInternals.getKey(),
             Collections.singletonList(timer.getNamespace()))));
   }
+
+  @Override
+  public void close() throws Exception {
+    super.close();
+
+    executorService.shutdown();
+
+    long shutdownTimeout = Duration.standardSeconds(10).getMillis();
+    try {
+      if (!executorService.awaitTermination(shutdownTimeout, TimeUnit.MILLISECONDS)) {
+        LOG.debug("The scheduled executor service did not properly terminate. Shutting "
+            + "it down now.");
+        executorService.shutdownNow();
+      }
+    } catch (InterruptedException e) {
+      LOG.debug("Could not properly await the termination of the scheduled executor service.", e);
+      executorService.shutdownNow();
+    }
+  }
+
 }

From 0066ca8d9bca3a5e2106548f2736351913c7f681 Mon Sep 17 00:00:00 2001
From: Thomas Weise <thw@apache.org>
Date: Tue, 2 May 2017 08:21:45 -0700
Subject: [PATCH 068/387] [BEAM-2180] update Apex version to 3.6.0

---
 runners/apex/pom.xml                                          | 4 ++--
 .../java/org/apache/beam/runners/apex/ApexRunnerTest.java     | 2 +-
 runners/apex/src/test/resources/beam-runners-apex.properties  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index aa4bddf57f76..85b15fa688d9 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <apex.core.version>3.5.0</apex.core.version>
+    <apex.core.version>3.6.0</apex.core.version>
     <apex.malhar.version>3.4.0</apex.malhar.version>
     <skipIntegrationTests>true</skipIntegrationTests>
     <!-- memory limit for embedded cluster -->
@@ -243,7 +243,7 @@
             </goals>
             <configuration>
               <ignoredUsedUndeclaredDependencies>
-                <ignoredUsedUndeclaredDependency>org.apache.apex:apex-api:jar:3.5.0</ignoredUsedUndeclaredDependency>
+                <ignoredUsedUndeclaredDependency>org.apache.apex:apex-api:jar:${apex.core.version}</ignoredUsedUndeclaredDependency>
                 <ignoredUsedUndeclaredDependency>org.apache.commons:commons-lang3::3.1</ignoredUsedUndeclaredDependency>
                 <ignoredUsedUndeclaredDependency>commons-io:commons-io:jar:2.4</ignoredUsedUndeclaredDependency>
                 <ignoredUsedUndeclaredDependency>com.esotericsoftware.kryo:kryo::${apex.kryo.version}</ignoredUsedUndeclaredDependency>
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
index 0fedc47fd055..e9e9a5b5c6a8 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
@@ -58,7 +58,7 @@ public void testConfigProperties() throws Exception {
     File tmp = File.createTempFile("beam-runners-apex-", ".properties");
     tmp.deleteOnExit();
     Properties props = new Properties();
-    props.setProperty("dt.operator." + operName + ".attr.MEMORY_MB", "64");
+    props.setProperty("apex.operator." + operName + ".attr.MEMORY_MB", "64");
     try (FileOutputStream fos = new FileOutputStream(tmp)) {
       props.store(fos, "");
     }
diff --git a/runners/apex/src/test/resources/beam-runners-apex.properties b/runners/apex/src/test/resources/beam-runners-apex.properties
index 48f8b05959b8..55cb397390c7 100644
--- a/runners/apex/src/test/resources/beam-runners-apex.properties
+++ b/runners/apex/src/test/resources/beam-runners-apex.properties
@@ -17,4 +17,4 @@
 ################################################################################
 
 # properties for unit test
-dt.operator.testProperties.attr.MEMORY_MB=32
+apex.operator.testProperties.attr.MEMORY_MB=32

From 7512a73cf8aa2a527c89ecb054e92207411ed241 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 4 May 2017 18:33:16 -0700
Subject: [PATCH 069/387] [BEAM-2016] Delete HdfsFileSource & Sink

---
 sdks/java/io/hadoop-file-system/README.md     |  43 --
 sdks/java/io/hadoop-file-system/pom.xml       |  24 -
 .../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 --------------
 .../beam/sdk/io/hdfs/HDFSFileSource.java      | 625 ------------------
 .../org/apache/beam/sdk/io/hdfs/Sink.java     | 195 ------
 .../apache/beam/sdk/io/hdfs/UGIHelper.java    |  38 --
 .../org/apache/beam/sdk/io/hdfs/Write.java    | 588 ----------------
 .../apache/beam/sdk/io/hdfs/package-info.java |   3 +-
 .../beam/sdk/io/hdfs/HDFSFileSinkTest.java    | 172 -----
 .../beam/sdk/io/hdfs/HDFSFileSourceTest.java  | 231 -------
 10 files changed, 2 insertions(+), 2395 deletions(-)
 delete mode 100644 sdks/java/io/hadoop-file-system/README.md
 delete mode 100644 sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java
 delete mode 100644 sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSource.java
 delete mode 100644 sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Sink.java
 delete mode 100644 sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/UGIHelper.java
 delete mode 100644 sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java
 delete mode 100644 sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSinkTest.java
 delete mode 100644 sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSourceTest.java

diff --git a/sdks/java/io/hadoop-file-system/README.md b/sdks/java/io/hadoop-file-system/README.md
deleted file mode 100644
index 3a734f214ae7..000000000000
--- a/sdks/java/io/hadoop-file-system/README.md
+++ /dev/null
@@ -1,43 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-# HDFS IO
-
-This library provides HDFS sources and sinks to make it possible to read and
-write Apache Hadoop file formats from Apache Beam pipelines.
-
-Currently, only the read path is implemented. A `HDFSFileSource` allows any
-Hadoop `FileInputFormat` to be read as a `PCollection`.
-
-A `HDFSFileSource` can be read from using the
-`org.apache.beam.sdk.io.Read` transform. For example:
-
-```java
-HDFSFileSource<K, V> source = HDFSFileSource.from(path, MyInputFormat.class,
-  MyKey.class, MyValue.class);
-PCollection<KV<MyKey, MyValue>> records = pipeline.apply(Read.from(mySource));
-```
-
-Alternatively, the `readFrom` method is a convenience method that returns a read
-transform. For example:
-
-```java
-PCollection<KV<MyKey, MyValue>> records = pipeline.apply(HDFSFileSource.readFrom(path,
-  MyInputFormat.class, MyKey.class, MyValue.class));
-```
diff --git a/sdks/java/io/hadoop-file-system/pom.xml b/sdks/java/io/hadoop-file-system/pom.xml
index 562277eeb116..3b392c28b27c 100644
--- a/sdks/java/io/hadoop-file-system/pom.xml
+++ b/sdks/java/io/hadoop-file-system/pom.xml
@@ -81,11 +81,6 @@
       <artifactId>beam-sdks-java-core</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-java-io-hadoop-common</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
@@ -123,25 +118,6 @@
       <artifactId>jsr305</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro-mapred</artifactId>
-      <version>${avro.version}</version>
-      <classifier>hadoop2</classifier>
-      <exclusions>
-        <!-- exclude old Jetty version of servlet API -->
-        <exclusion>
-          <groupId>org.mortbay.jetty</groupId>
-          <artifactId>servlet-api</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java
deleted file mode 100644
index aee73c4f0561..000000000000
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.hdfs;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import java.io.IOException;
-import java.net.URI;
-import java.security.PrivilegedExceptionAction;
-import java.util.Random;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.mapred.AvroKey;
-import org.apache.avro.mapreduce.AvroKeyOutputFormat;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.AvroCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.hadoop.SerializableConfiguration;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.values.KV;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskID;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.mapreduce.task.JobContextImpl;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-
-/**
- * A {@link Sink} for writing records to a Hadoop filesystem using a Hadoop file-based
- * output
- * format.
- *
- * <p>To write a {@link org.apache.beam.sdk.values.PCollection} of elements of type T to Hadoop
- * filesystem use {@link HDFSFileSink#to}, specify the path (this can be any Hadoop supported
- * filesystem: HDFS, S3, GCS etc), the Hadoop {@link FileOutputFormat}, the key class K and the
- * value class V and finally the {@link SerializableFunction} to map from T to {@link KV} of K
- * and V.
- *
- * <p>{@code HDFSFileSink} can be used by {@link Write} to create write
- * transform. See example below.
- *
- * <p>{@code HDFSFileSink} comes with helper methods to write text and Apache Avro. For example:
- *
- * <pre>
- * {@code
- * HDFSFileSink<CustomSpecificAvroClass, AvroKey<CustomSpecificAvroClass>, NullWritable> sink =
- *   HDFSFileSink.toAvro(path, AvroCoder.of(CustomSpecificAvroClass.class));
- * avroRecordsPCollection.apply(Write.to(sink));
- * }
- * </pre>
- *
- * @param <T> the type of elements of the input {@link org.apache.beam.sdk.values.PCollection}.
- * @param <K> the type of keys to be written to the sink via {@link FileOutputFormat}.
- * @param <V> the type of values to be written to the sink via {@link FileOutputFormat}.
- */
-@AutoValue
-@Experimental
-public abstract class HDFSFileSink<T, K, V> extends Sink<T> {
-
-  private static final JobID jobId = new JobID(
-      Long.toString(System.currentTimeMillis()),
-      new Random().nextInt(Integer.MAX_VALUE));
-
-  public abstract String path();
-  public abstract Class<? extends FileOutputFormat<K, V>> formatClass();
-  public abstract Class<K> keyClass();
-  public abstract Class<V> valueClass();
-  public abstract SerializableFunction<T, KV<K, V>> outputConverter();
-  public abstract SerializableConfiguration serializableConfiguration();
-  public @Nullable abstract String username();
-  public abstract boolean validate();
-
-  // =======================================================================
-  // Factory methods
-  // =======================================================================
-
-  public static <T, K, V, W extends FileOutputFormat<K, V>> HDFSFileSink<T, K, V>
-  to(String path,
-     Class<W> formatClass,
-     Class<K> keyClass,
-     Class<V> valueClass,
-     SerializableFunction<T, KV<K, V>> outputConverter) {
-    return HDFSFileSink.<T, K, V>builder()
-        .setPath(path)
-        .setFormatClass(formatClass)
-        .setKeyClass(keyClass)
-        .setValueClass(valueClass)
-        .setOutputConverter(outputConverter)
-        .setConfiguration(null)
-        .setUsername(null)
-        .setValidate(true)
-        .build();
-  }
-
-  public static <T> HDFSFileSink<T, NullWritable, Text> toText(String path) {
-    SerializableFunction<T, KV<NullWritable, Text>> outputConverter =
-        new SerializableFunction<T, KV<NullWritable, Text>>() {
-          @Override
-          public KV<NullWritable, Text> apply(T input) {
-            return KV.of(NullWritable.get(), new Text(input.toString()));
-          }
-        };
-    return to(path, TextOutputFormat.class, NullWritable.class, Text.class, outputConverter);
-  }
-
-  /**
-   * Helper to create Avro sink given {@link AvroCoder}. Keep in mind that configuration
-   * object is altered to enable Avro output.
-   */
-  public static <T> HDFSFileSink<T, AvroKey<T>, NullWritable> toAvro(String path,
-                                                                     final AvroCoder<T> coder,
-                                                                     Configuration conf) {
-    SerializableFunction<T, KV<AvroKey<T>, NullWritable>> outputConverter =
-        new SerializableFunction<T, KV<AvroKey<T>, NullWritable>>() {
-          @Override
-          public KV<AvroKey<T>, NullWritable> apply(T input) {
-            return KV.of(new AvroKey<>(input), NullWritable.get());
-          }
-        };
-    conf.set("avro.schema.output.key", coder.getSchema().toString());
-    return to(
-        path,
-        AvroKeyOutputFormat.class,
-        (Class<AvroKey<T>>) (Class<?>) AvroKey.class,
-        NullWritable.class,
-        outputConverter).withConfiguration(conf);
-  }
-
-  /**
-   * Helper to create Avro sink given {@link Schema}. Keep in mind that configuration
-   * object is altered to enable Avro output.
-   */
-  public static HDFSFileSink<GenericRecord, AvroKey<GenericRecord>, NullWritable>
-  toAvro(String path, Schema schema, Configuration conf) {
-    return toAvro(path, AvroCoder.of(schema), conf);
-  }
-
-  /**
-   * Helper to create Avro sink given {@link Class}. Keep in mind that configuration
-   * object is altered to enable Avro output.
-   */
-  public static <T> HDFSFileSink<T, AvroKey<T>, NullWritable> toAvro(String path,
-                                                                     Class<T> cls,
-                                                                     Configuration conf) {
-    return toAvro(path, AvroCoder.of(cls), conf);
-  }
-
-  // =======================================================================
-  // Builder methods
-  // =======================================================================
-
-  public abstract Builder<T, K, V> toBuilder();
-  public static <T, K, V> Builder builder() {
-    return new AutoValue_HDFSFileSink.Builder<>();
-  }
-
-  /**
-   * AutoValue builder for {@link HDFSFileSink}.
-   */
-  @AutoValue.Builder
-  public abstract static class Builder<T, K, V> {
-    public abstract Builder<T, K, V> setPath(String path);
-    public abstract Builder<T, K, V> setFormatClass(
-        Class<? extends FileOutputFormat<K, V>> formatClass);
-    public abstract Builder<T, K, V> setKeyClass(Class<K> keyClass);
-    public abstract Builder<T, K, V> setValueClass(Class<V> valueClass);
-    public abstract Builder<T, K, V> setOutputConverter(
-        SerializableFunction<T, KV<K, V>> outputConverter);
-    public abstract Builder<T, K, V> setSerializableConfiguration(
-        SerializableConfiguration serializableConfiguration);
-    public Builder<T, K, V> setConfiguration(@Nullable Configuration configuration) {
-      if (configuration == null) {
-        configuration = new Configuration(false);
-      }
-      return this.setSerializableConfiguration(new SerializableConfiguration(configuration));
-    }
-    public abstract Builder<T, K, V> setUsername(String username);
-    public abstract Builder<T, K, V> setValidate(boolean validate);
-    public abstract HDFSFileSink<T, K, V> build();
-  }
-
-  public HDFSFileSink<T, K, V> withConfiguration(@Nullable Configuration configuration) {
-    return this.toBuilder().setConfiguration(configuration).build();
-  }
-
-  public HDFSFileSink<T, K, V> withUsername(@Nullable String username) {
-    return this.toBuilder().setUsername(username).build();
-  }
-
-  // =======================================================================
-  // Sink
-  // =======================================================================
-
-  @Override
-  public void validate(PipelineOptions options) {
-    if (validate()) {
-      try {
-        UGIHelper.getBestUGI(username()).doAs(new PrivilegedExceptionAction<Void>() {
-          @Override
-          public Void run() throws Exception {
-            FileSystem fs = FileSystem.get(new URI(path()),
-                SerializableConfiguration.newConfiguration(serializableConfiguration()));
-            checkState(!fs.exists(new Path(path())), "Output path %s already exists", path());
-            return null;
-          }
-        });
-      } catch (IOException | InterruptedException e) {
-        throw new RuntimeException(e);
-      }
-    }
-  }
-
-  @Override
-  public Sink.WriteOperation<T, String> createWriteOperation() {
-    return new HDFSWriteOperation<>(this, path(), formatClass());
-  }
-
-  private Job newJob() throws IOException {
-    Job job = SerializableConfiguration.newJob(serializableConfiguration());
-    job.setJobID(jobId);
-    job.setOutputKeyClass(keyClass());
-    job.setOutputValueClass(valueClass());
-    return job;
-  }
-
-  // =======================================================================
-  // WriteOperation
-  // =======================================================================
-
-  /** {{@link WriteOperation}} for HDFS. */
-  private static class HDFSWriteOperation<T, K, V> extends WriteOperation<T, String> {
-
-    private final HDFSFileSink<T, K, V> sink;
-    private final String path;
-    private final Class<? extends FileOutputFormat<K, V>> formatClass;
-
-    HDFSWriteOperation(HDFSFileSink<T, K, V> sink,
-                       String path,
-                       Class<? extends FileOutputFormat<K, V>> formatClass) {
-      this.sink = sink;
-      this.path = path;
-      this.formatClass = formatClass;
-    }
-
-    @Override
-    public void initialize(PipelineOptions options) throws Exception {
-      Job job = sink.newJob();
-      FileOutputFormat.setOutputPath(job, new Path(path));
-    }
-
-    @Override
-    public void setWindowedWrites(boolean windowedWrites) {
-    }
-
-    @Override
-    public void finalize(final Iterable<String> writerResults, PipelineOptions options)
-        throws Exception {
-      UGIHelper.getBestUGI(sink.username()).doAs(new PrivilegedExceptionAction<Void>() {
-        @Override
-        public Void run() throws Exception {
-          doFinalize(writerResults);
-          return null;
-        }
-      });
-    }
-
-    private void doFinalize(Iterable<String> writerResults) throws Exception {
-      Job job = sink.newJob();
-      FileSystem fs = FileSystem.get(new URI(path), job.getConfiguration());
-      // If there are 0 output shards, just create output folder.
-      if (!writerResults.iterator().hasNext()) {
-        fs.mkdirs(new Path(path));
-        return;
-      }
-
-      // job successful
-      JobContext context = new JobContextImpl(job.getConfiguration(), job.getJobID());
-      FileOutputCommitter outputCommitter = new FileOutputCommitter(new Path(path), context);
-      outputCommitter.commitJob(context);
-
-      // get actual output shards
-      Set<String> actual = Sets.newHashSet();
-      FileStatus[] statuses = fs.listStatus(new Path(path), new PathFilter() {
-        @Override
-        public boolean accept(Path path) {
-          String name = path.getName();
-          return !name.startsWith("_") && !name.startsWith(".");
-        }
-      });
-
-      // get expected output shards
-      Set<String> expected = Sets.newHashSet(writerResults);
-      checkState(
-          expected.size() == Lists.newArrayList(writerResults).size(),
-          "Data loss due to writer results hash collision");
-      for (FileStatus s : statuses) {
-        String name = s.getPath().getName();
-        int pos = name.indexOf('.');
-        actual.add(pos > 0 ? name.substring(0, pos) : name);
-      }
-
-      checkState(actual.equals(expected), "Writer results and output files do not match");
-
-      // rename output shards to Hadoop style, i.e. part-r-00000.txt
-      int i = 0;
-      for (FileStatus s : statuses) {
-        String name = s.getPath().getName();
-        int pos = name.indexOf('.');
-        String ext = pos > 0 ? name.substring(pos) : "";
-        fs.rename(
-            s.getPath(),
-            new Path(s.getPath().getParent(), String.format("part-r-%05d%s", i, ext)));
-        i++;
-      }
-    }
-
-    @Override
-    public Writer<T, String> createWriter(PipelineOptions options) throws Exception {
-      return new HDFSWriter<>(this, path, formatClass);
-    }
-
-    @Override
-    public Sink<T> getSink() {
-      return sink;
-    }
-
-    @Override
-    public Coder<String> getWriterResultCoder() {
-      return StringUtf8Coder.of();
-    }
-
-  }
-
-  // =======================================================================
-  // Writer
-  // =======================================================================
-
-  private static class HDFSWriter<T, K, V> extends Writer<T, String> {
-
-    private final HDFSWriteOperation<T, K, V> writeOperation;
-    private final String path;
-    private final Class<? extends FileOutputFormat<K, V>> formatClass;
-
-    // unique hash for each task
-    private int hash;
-
-    private TaskAttemptContext context;
-    private RecordWriter<K, V> recordWriter;
-    private FileOutputCommitter outputCommitter;
-
-    HDFSWriter(HDFSWriteOperation<T, K, V> writeOperation,
-               String path,
-               Class<? extends FileOutputFormat<K, V>> formatClass) {
-      this.writeOperation = writeOperation;
-      this.path = path;
-      this.formatClass = formatClass;
-    }
-
-    @Override
-    public void openWindowed(final String uId,
-                             BoundedWindow window,
-                             PaneInfo paneInfo,
-                             int shard,
-                             int numShards) throws Exception {
-      throw new UnsupportedOperationException("Windowing support not implemented yet for"
-          + "HDFS. Window " + window);
-    }
-
-    @Override
-    public void openUnwindowed(final String uId, int shard, int numShards) throws Exception {
-      UGIHelper.getBestUGI(writeOperation.sink.username()).doAs(
-          new PrivilegedExceptionAction<Void>() {
-            @Override
-            public Void run() throws Exception {
-              doOpen(uId);
-              return null;
-            }
-          }
-      );
-    }
-
-    private void doOpen(String uId) throws Exception {
-      this.hash = uId.hashCode();
-
-      Job job = writeOperation.sink.newJob();
-      FileOutputFormat.setOutputPath(job, new Path(path));
-
-      // Each Writer is responsible for writing one bundle of elements and is represented by one
-      // unique Hadoop task based on uId/hash. All tasks share the same job ID.
-      JobID jobId = job.getJobID();
-      TaskID taskId = new TaskID(jobId, TaskType.REDUCE, hash);
-      context = new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID(taskId, 0));
-
-      FileOutputFormat<K, V> outputFormat = formatClass.newInstance();
-      recordWriter = outputFormat.getRecordWriter(context);
-      outputCommitter = (FileOutputCommitter) outputFormat.getOutputCommitter(context);
-    }
-
-    @Override
-    public void write(T value) throws Exception {
-      checkNotNull(recordWriter,
-          "Record writer can't be null. Make sure to open Writer first!");
-      KV<K, V> kv = writeOperation.sink.outputConverter().apply(value);
-      recordWriter.write(kv.getKey(), kv.getValue());
-    }
-
-    @Override
-    public void cleanup() throws Exception {
-
-    }
-
-    @Override
-    public String close() throws Exception {
-      return UGIHelper.getBestUGI(writeOperation.sink.username()).doAs(
-          new PrivilegedExceptionAction<String>() {
-            @Override
-            public String run() throws Exception {
-              return doClose();
-            }
-          });
-    }
-
-    private String doClose() throws Exception {
-      // task/attempt successful
-      recordWriter.close(context);
-      outputCommitter.commitTask(context);
-
-      // result is prefix of the output file name
-      return String.format("part-r-%d", hash);
-    }
-
-    @Override
-    public WriteOperation<T, String> getWriteOperation() {
-      return writeOperation;
-    }
-
-  }
-
-}
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSource.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSource.java
deleted file mode 100644
index 5cc20978628a..000000000000
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSource.java
+++ /dev/null
@@ -1,625 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.hdfs;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.security.PrivilegedExceptionAction;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.NoSuchElementException;
-import javax.annotation.Nullable;
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.mapred.AvroKey;
-import org.apache.avro.mapreduce.AvroKeyInputFormat;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.AvroCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.hadoop.SerializableConfiguration;
-import org.apache.beam.sdk.io.hadoop.WritableCoder;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.values.KV;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A {@code BoundedSource} for reading files resident in a Hadoop filesystem using a
- * Hadoop file-based input format.
- *
- * <p>To read a {@link org.apache.beam.sdk.values.PCollection} of
- * {@link org.apache.beam.sdk.values.KV} key-value pairs from one or more
- * Hadoop files, use {@link HDFSFileSource#from} to specify the path(s) of the files to
- * read, the Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}, the
- * key class and the value class.
- *
- * <p>A {@code HDFSFileSource} can be read from using the
- * {@link org.apache.beam.sdk.io.Read} transform. For example:
- *
- * <pre>
- * {@code
- * HDFSFileSource<K, V> source = HDFSFileSource.from(path, MyInputFormat.class,
- *   MyKey.class, MyValue.class);
- * PCollection<KV<MyKey, MyValue>> records = pipeline.apply(Read.from(mySource));
- * }
- * </pre>
- *
- * <p>Implementation note: Since Hadoop's
- * {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}
- * determines the input splits, this class extends {@link BoundedSource} rather than
- * {@link org.apache.beam.sdk.io.OffsetBasedSource}, since the latter
- * dictates input splits.
- * @param <T> the type of elements of the result {@link org.apache.beam.sdk.values.PCollection}.
- * @param <K> the type of keys to be read from the source via {@link FileInputFormat}.
- * @param <V> the type of values to be read from the source via {@link FileInputFormat}.
- */
-@AutoValue
-@Experimental
-public abstract class HDFSFileSource<T, K, V> extends BoundedSource<T> {
-  private static final long serialVersionUID = 0L;
-
-  private static final Logger LOG = LoggerFactory.getLogger(HDFSFileSource.class);
-
-  public abstract String filepattern();
-  public abstract Class<? extends FileInputFormat<K, V>> formatClass();
-  public abstract Coder<T> coder();
-  public abstract SerializableFunction<KV<K, V>, T> inputConverter();
-  public abstract SerializableConfiguration serializableConfiguration();
-  public @Nullable abstract SerializableSplit serializableSplit();
-  public @Nullable abstract String username();
-  public abstract boolean validateSource();
-
-  // =======================================================================
-  // Factory methods
-  // =======================================================================
-
-  public static <T, K, V, W extends FileInputFormat<K, V>> HDFSFileSource<T, K, V>
-  from(String filepattern,
-       Class<W> formatClass,
-       Coder<T> coder,
-       SerializableFunction<KV<K, V>, T> inputConverter) {
-    return HDFSFileSource.<T, K, V>builder()
-        .setFilepattern(filepattern)
-        .setFormatClass(formatClass)
-        .setCoder(coder)
-        .setInputConverter(inputConverter)
-        .setConfiguration(null)
-        .setUsername(null)
-        .setValidateSource(true)
-        .setSerializableSplit(null)
-        .build();
-  }
-
-  public static <K, V, W extends FileInputFormat<K, V>> HDFSFileSource<KV<K, V>, K, V>
-  from(String filepattern,
-       Class<W> formatClass,
-       Class<K> keyClass,
-       Class<V> valueClass) {
-    KvCoder<K, V> coder = KvCoder.of(getDefaultCoder(keyClass), getDefaultCoder(valueClass));
-    SerializableFunction<KV<K, V>, KV<K, V>> inputConverter =
-        new SerializableFunction<KV<K, V>, KV<K, V>>() {
-          @Override
-          public KV<K, V> apply(KV<K, V> input) {
-            return input;
-          }
-        };
-    return HDFSFileSource.<KV<K, V>, K, V>builder()
-        .setFilepattern(filepattern)
-        .setFormatClass(formatClass)
-        .setCoder(coder)
-        .setInputConverter(inputConverter)
-        .setConfiguration(null)
-        .setUsername(null)
-        .setValidateSource(true)
-        .setSerializableSplit(null)
-        .build();
-  }
-
-  public static HDFSFileSource<String, LongWritable, Text>
-  fromText(String filepattern) {
-    SerializableFunction<KV<LongWritable, Text>, String> inputConverter =
-        new SerializableFunction<KV<LongWritable, Text>, String>() {
-      @Override
-      public String apply(KV<LongWritable, Text> input) {
-        return input.getValue().toString();
-      }
-    };
-    return from(filepattern, TextInputFormat.class, StringUtf8Coder.of(), inputConverter);
-  }
-
-  /**
-   * Helper to read from Avro source given {@link AvroCoder}. Keep in mind that configuration
-   * object is altered to enable Avro input.
-   */
-  public static <T> HDFSFileSource<T, AvroKey<T>, NullWritable>
-  fromAvro(String filepattern, final AvroCoder<T> coder, Configuration conf) {
-    Class<AvroKeyInputFormat<T>> formatClass = castClass(AvroKeyInputFormat.class);
-    SerializableFunction<KV<AvroKey<T>, NullWritable>, T> inputConverter =
-        new SerializableFunction<KV<AvroKey<T>, NullWritable>, T>() {
-          @Override
-          public T apply(KV<AvroKey<T>, NullWritable> input) {
-            try {
-              return CoderUtils.clone(coder, input.getKey().datum());
-            } catch (CoderException e) {
-              throw new RuntimeException(e);
-            }
-          }
-        };
-    conf.set("avro.schema.input.key", coder.getSchema().toString());
-    return from(filepattern, formatClass, coder, inputConverter).withConfiguration(conf);
-  }
-
-  /**
-   * Helper to read from Avro source given {@link Schema}. Keep in mind that configuration
-   * object is altered to enable Avro input.
-   */
-  public static HDFSFileSource<GenericRecord, AvroKey<GenericRecord>, NullWritable>
-  fromAvro(String filepattern, Schema schema, Configuration conf) {
-    return fromAvro(filepattern, AvroCoder.of(schema), conf);
-  }
-
-  /**
-   * Helper to read from Avro source given {@link Class}. Keep in mind that configuration
-   * object is altered to enable Avro input.
-   */
-  public static <T> HDFSFileSource<T, AvroKey<T>, NullWritable>
-  fromAvro(String filepattern, Class<T> cls, Configuration conf) {
-    return fromAvro(filepattern, AvroCoder.of(cls), conf);
-  }
-
-  // =======================================================================
-  // Builder methods
-  // =======================================================================
-
-  public abstract HDFSFileSource.Builder<T, K, V> toBuilder();
-  public static <T, K, V> HDFSFileSource.Builder builder() {
-    return new AutoValue_HDFSFileSource.Builder<>();
-  }
-
-  /**
-   * AutoValue builder for {@link HDFSFileSource}.
-   */
-  @AutoValue.Builder
-  public abstract static class Builder<T, K, V> {
-    public abstract Builder<T, K, V> setFilepattern(String filepattern);
-    public abstract Builder<T, K, V> setFormatClass(
-        Class<? extends FileInputFormat<K, V>> formatClass);
-    public abstract Builder<T, K, V> setCoder(Coder<T> coder);
-    public abstract Builder<T, K, V> setInputConverter(
-        SerializableFunction<KV<K, V>, T> inputConverter);
-    public abstract Builder<T, K, V> setSerializableConfiguration(
-        SerializableConfiguration serializableConfiguration);
-    public Builder<T, K, V> setConfiguration(Configuration configuration) {
-      if (configuration == null) {
-        configuration = new Configuration(false);
-      }
-      return this.setSerializableConfiguration(new SerializableConfiguration(configuration));
-    }
-    public abstract Builder<T, K, V> setSerializableSplit(SerializableSplit serializableSplit);
-    public abstract Builder<T, K, V> setUsername(@Nullable String username);
-    public abstract Builder<T, K, V> setValidateSource(boolean validate);
-    public abstract HDFSFileSource<T, K, V> build();
-  }
-
-  public HDFSFileSource<T, K, V> withConfiguration(@Nullable Configuration configuration) {
-    return this.toBuilder().setConfiguration(configuration).build();
-  }
-
-  public HDFSFileSource<T, K, V> withUsername(@Nullable String username) {
-    return this.toBuilder().setUsername(username).build();
-  }
-
-  // =======================================================================
-  // BoundedSource
-  // =======================================================================
-
-  @Override
-  public List<? extends BoundedSource<T>> split(
-      final long desiredBundleSizeBytes,
-      PipelineOptions options) throws Exception {
-    if (serializableSplit() == null) {
-      List<InputSplit> inputSplits = UGIHelper.getBestUGI(username()).doAs(
-          new PrivilegedExceptionAction<List<InputSplit>>() {
-            @Override
-            public List<InputSplit> run() throws Exception {
-              return computeSplits(desiredBundleSizeBytes, serializableConfiguration());
-            }
-          });
-      return Lists.transform(inputSplits,
-          new Function<InputSplit, BoundedSource<T>>() {
-            @Override
-            public BoundedSource<T> apply(@Nullable InputSplit inputSplit) {
-              SerializableSplit serializableSplit = new SerializableSplit(inputSplit);
-              return HDFSFileSource.this.toBuilder()
-                  .setSerializableSplit(serializableSplit)
-                  .build();
-            }
-          });
-    } else {
-      return ImmutableList.of(this);
-    }
-  }
-
-  @Override
-  public long getEstimatedSizeBytes(PipelineOptions options) {
-    long size = 0;
-
-    try {
-      // If this source represents a split from split,
-      // then return the size of the split, rather then the entire input
-      if (serializableSplit() != null) {
-        return serializableSplit().getSplit().getLength();
-      }
-
-      size += UGIHelper.getBestUGI(username()).doAs(new PrivilegedExceptionAction<Long>() {
-        @Override
-        public Long run() throws Exception {
-          long size = 0;
-          Job job = SerializableConfiguration.newJob(serializableConfiguration());
-          for (FileStatus st : listStatus(createFormat(job), job)) {
-            size += st.getLen();
-          }
-          return size;
-        }
-      });
-    } catch (IOException e) {
-      LOG.warn(
-          "Will estimate size of input to be 0 bytes. Can't estimate size of the input due to:", e);
-      // ignore, and return 0
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      LOG.warn(
-          "Will estimate size of input to be 0 bytes. Can't estimate size of the input due to:", e);
-      // ignore, and return 0
-    }
-    return size;
-  }
-
-  @Override
-  public BoundedReader<T> createReader(PipelineOptions options) throws IOException {
-    this.validate();
-    return new HDFSFileReader<>(this, filepattern(), formatClass(), serializableSplit());
-  }
-
-  @Override
-  public void validate() {
-    if (validateSource()) {
-      try {
-        UGIHelper.getBestUGI(username()).doAs(new PrivilegedExceptionAction<Void>() {
-              @Override
-              public Void run() throws Exception {
-                final Path pathPattern = new Path(filepattern());
-                FileSystem fs = FileSystem.get(pathPattern.toUri(),
-                    SerializableConfiguration.newConfiguration(serializableConfiguration()));
-                FileStatus[] fileStatuses = fs.globStatus(pathPattern);
-                checkState(
-                    fileStatuses != null && fileStatuses.length > 0,
-                    "Unable to find any files matching %s", filepattern());
-                  return null;
-                }
-              });
-      } catch (IOException | InterruptedException e) {
-        throw new RuntimeException(e);
-      }
-    }
-  }
-
-  @Override
-  public Coder<T> getDefaultOutputCoder() {
-    return coder();
-  }
-
-  // =======================================================================
-  // Helpers
-  // =======================================================================
-
-  private List<InputSplit> computeSplits(long desiredBundleSizeBytes,
-                                         SerializableConfiguration serializableConfiguration)
-      throws IOException, IllegalAccessException, InstantiationException {
-    Job job = SerializableConfiguration.newJob(serializableConfiguration);
-    FileInputFormat.setMinInputSplitSize(job, desiredBundleSizeBytes);
-    FileInputFormat.setMaxInputSplitSize(job, desiredBundleSizeBytes);
-    return createFormat(job).getSplits(job);
-  }
-
-  private FileInputFormat<K, V> createFormat(Job job)
-      throws IOException, IllegalAccessException, InstantiationException {
-    Path path = new Path(filepattern());
-    FileInputFormat.addInputPath(job, path);
-    return formatClass().newInstance();
-  }
-
-  private List<FileStatus> listStatus(FileInputFormat<K, V> format, Job job)
-      throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
-    // FileInputFormat#listStatus is protected, so call using reflection
-    Method listStatus = FileInputFormat.class.getDeclaredMethod("listStatus", JobContext.class);
-    listStatus.setAccessible(true);
-    @SuppressWarnings("unchecked")
-    List<FileStatus> stat = (List<FileStatus>) listStatus.invoke(format, job);
-    return stat;
-  }
-
-  @SuppressWarnings("unchecked")
-  private static <T> Coder<T> getDefaultCoder(Class<T> c) {
-    if (Writable.class.isAssignableFrom(c)) {
-      Class<? extends Writable> writableClass = (Class<? extends Writable>) c;
-      return (Coder<T>) WritableCoder.of(writableClass);
-    } else if (Void.class.equals(c)) {
-      return (Coder<T>) VoidCoder.of();
-    }
-    // TODO: how to use registered coders here?
-    throw new IllegalStateException("Cannot find coder for " + c);
-  }
-
-  @SuppressWarnings("unchecked")
-  private static <T> Class<T> castClass(Class<?> aClass) {
-    return (Class<T>) aClass;
-  }
-
-  // =======================================================================
-  // BoundedReader
-  // =======================================================================
-
-  private static class HDFSFileReader<T, K, V> extends BoundedSource.BoundedReader<T> {
-
-    private final HDFSFileSource<T, K, V> source;
-    private final String filepattern;
-    private final Class<? extends FileInputFormat<K, V>> formatClass;
-    private final Job job;
-
-    private List<InputSplit> splits;
-    private ListIterator<InputSplit> splitsIterator;
-
-    private Configuration conf;
-    private FileInputFormat<?, ?> format;
-    private TaskAttemptContext attemptContext;
-    private RecordReader<K, V> currentReader;
-    private KV<K, V> currentPair;
-
-    HDFSFileReader(
-        HDFSFileSource<T, K, V> source,
-        String filepattern,
-        Class<? extends FileInputFormat<K, V>> formatClass,
-        SerializableSplit serializableSplit)
-        throws IOException {
-      this.source = source;
-      this.filepattern = filepattern;
-      this.formatClass = formatClass;
-      this.job = SerializableConfiguration.newJob(source.serializableConfiguration());
-
-      if (serializableSplit != null) {
-        this.splits = ImmutableList.of(serializableSplit.getSplit());
-        this.splitsIterator = splits.listIterator();
-      }
-    }
-
-    @Override
-    public boolean start() throws IOException {
-      Path path = new Path(filepattern);
-      FileInputFormat.addInputPath(job, path);
-
-      conf = job.getConfiguration();
-      try {
-        format = formatClass.newInstance();
-      } catch (InstantiationException | IllegalAccessException e) {
-        throw new IOException("Cannot instantiate file input format " + formatClass, e);
-      }
-      attemptContext = new TaskAttemptContextImpl(conf, new TaskAttemptID());
-
-      if (splitsIterator == null) {
-        splits = format.getSplits(job);
-        splitsIterator = splits.listIterator();
-      }
-
-      return advance();
-    }
-
-    @Override
-    public boolean advance() throws IOException {
-      try {
-        if (currentReader != null && currentReader.nextKeyValue()) {
-          currentPair = nextPair();
-          return true;
-        } else {
-          while (splitsIterator.hasNext()) {
-            // advance the reader and see if it has records
-            final InputSplit nextSplit = splitsIterator.next();
-            @SuppressWarnings("unchecked")
-            RecordReader<K, V> reader =
-                (RecordReader<K, V>) format.createRecordReader(nextSplit, attemptContext);
-            if (currentReader != null) {
-              currentReader.close();
-            }
-            currentReader = reader;
-            UGIHelper.getBestUGI(source.username()).doAs(new PrivilegedExceptionAction<Void>() {
-              @Override
-              public Void run() throws Exception {
-                currentReader.initialize(nextSplit, attemptContext);
-                return null;
-              }
-            });
-            if (currentReader.nextKeyValue()) {
-              currentPair = nextPair();
-              return true;
-            }
-            currentReader.close();
-            currentReader = null;
-          }
-          // either no next split or all readers were empty
-          currentPair = null;
-          return false;
-        }
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        throw new IOException(e);
-      }
-    }
-
-    @Override
-    public T getCurrent() throws NoSuchElementException {
-      if (currentPair == null) {
-        throw new NoSuchElementException();
-      }
-      return source.inputConverter().apply(currentPair);
-    }
-
-    @Override
-    public void close() throws IOException {
-      if (currentReader != null) {
-        currentReader.close();
-        currentReader = null;
-      }
-      currentPair = null;
-    }
-
-    @Override
-    public BoundedSource<T> getCurrentSource() {
-      return source;
-    }
-
-    @SuppressWarnings("unchecked")
-    private KV<K, V> nextPair() throws IOException, InterruptedException {
-      K key = currentReader.getCurrentKey();
-      V value = currentReader.getCurrentValue();
-      // clone Writable objects since they are reused between calls to RecordReader#nextKeyValue
-      if (key instanceof Writable) {
-        key = (K) WritableUtils.clone((Writable) key, conf);
-      }
-      if (value instanceof Writable) {
-        value = (V) WritableUtils.clone((Writable) value, conf);
-      }
-      return KV.of(key, value);
-    }
-
-    // =======================================================================
-    // Optional overrides
-    // =======================================================================
-
-    @Override
-    public Double getFractionConsumed() {
-      if (currentReader == null) {
-        return 0.0;
-      }
-      if (splits.isEmpty()) {
-        return 1.0;
-      }
-      int index = splitsIterator.previousIndex();
-      int numReaders = splits.size();
-      if (index == numReaders) {
-        return 1.0;
-      }
-      double before = 1.0 * index / numReaders;
-      double after = 1.0 * (index + 1) / numReaders;
-      Double fractionOfCurrentReader = getProgress();
-      if (fractionOfCurrentReader == null) {
-        return before;
-      }
-      return before + fractionOfCurrentReader * (after - before);
-    }
-
-    private Double getProgress() {
-      try {
-        return (double) currentReader.getProgress();
-      } catch (IOException | InterruptedException e) {
-        return null;
-      }
-    }
-
-  }
-
-  // =======================================================================
-  // SerializableSplit
-  // =======================================================================
-
-  /**
-   * A wrapper to allow Hadoop {@link org.apache.hadoop.mapreduce.InputSplit}s to be
-   * serialized using Java's standard serialization mechanisms. Note that the InputSplit
-   * has to be Writable (which most are).
-   */
-  protected static class SerializableSplit implements Externalizable {
-    private static final long serialVersionUID = 0L;
-
-    private InputSplit split;
-
-    public SerializableSplit() {
-    }
-
-    public SerializableSplit(InputSplit split) {
-      checkArgument(split instanceof Writable, "Split is not writable: %s", split);
-      this.split = split;
-    }
-
-    public InputSplit getSplit() {
-      return split;
-    }
-
-    @Override
-    public void writeExternal(ObjectOutput out) throws IOException {
-      out.writeUTF(split.getClass().getCanonicalName());
-      ((Writable) split).write(out);
-    }
-
-    @Override
-    public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-      String className = in.readUTF();
-      try {
-        split = (InputSplit) Class.forName(className).newInstance();
-        ((Writable) split).readFields(in);
-      } catch (InstantiationException | IllegalAccessException e) {
-        throw new IOException(e);
-      }
-    }
-  }
-
-}
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Sink.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Sink.java
deleted file mode 100644
index fe2db5f37507..000000000000
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Sink.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.hdfs;
-
-import java.io.Serializable;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.display.HasDisplayData;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-
-/**
- * This class is deprecated, and only exists for HDFSFileSink.
- */
-@Deprecated
-public abstract class Sink<T> implements Serializable, HasDisplayData {
-  /**
-   * Ensures that the sink is valid and can be written to before the write operation begins. One
-   * should use {@link com.google.common.base.Preconditions} to implement this method.
-   */
-  public abstract void validate(PipelineOptions options);
-
-  /**
-   * Returns an instance of a {@link WriteOperation} that can write to this Sink.
-   */
-  public abstract WriteOperation<T, ?> createWriteOperation();
-
-  /**
-   * {@inheritDoc}
-   *
-   * <p>By default, does not register any display data. Implementors may override this method
-   * to provide their own display data.
-   */
-  @Override
-  public void populateDisplayData(DisplayData.Builder builder) {}
-
-  /**
-   * A {@link WriteOperation} defines the process of a parallel write of objects to a Sink.
-   *
-   * <p>The {@code WriteOperation} defines how to perform initialization and finalization of a
-   * parallel write to a sink as well as how to create a {@link Sink.Writer} object that can write
-   * a bundle to the sink.
-   *
-   * <p>Since operations in Beam may be run multiple times for redundancy or fault-tolerance,
-   * the initialization and finalization defined by a WriteOperation <b>must be idempotent</b>.
-   *
-   * <p>{@code WriteOperation}s may be mutable; a {@code WriteOperation} is serialized after the
-   * call to {@code initialize} method and deserialized before calls to
-   * {@code createWriter} and {@code finalized}. However, it is not
-   * reserialized after {@code createWriter}, so {@code createWriter} should not mutate the
-   * state of the {@code WriteOperation}.
-   *
-   * <p>See {@link Sink} for more detailed documentation about the process of writing to a Sink.
-   *
-   * @param <T> The type of objects to write
-   * @param <WriteT> The result of a per-bundle write
-   */
-  public abstract static class WriteOperation<T, WriteT> implements Serializable {
-    /**
-     * Performs initialization before writing to the sink. Called before writing begins.
-     */
-    public abstract void initialize(PipelineOptions options) throws Exception;
-
-    /**
-     * Indicates that the operation will be performing windowed writes.
-     */
-    public abstract void setWindowedWrites(boolean windowedWrites);
-
-    /**
-     * Given an Iterable of results from bundle writes, performs finalization after writing and
-     * closes the sink. Called after all bundle writes are complete.
-     *
-     * <p>The results that are passed to finalize are those returned by bundles that completed
-     * successfully. Although bundles may have been run multiple times (for fault-tolerance), only
-     * one writer result will be passed to finalize for each bundle. An implementation of finalize
-     * should perform clean up of any failed and successfully retried bundles.  Note that these
-     * failed bundles will not have their writer result passed to finalize, so finalize should be
-     * capable of locating any temporary/partial output written by failed bundles.
-     *
-     * <p>A best practice is to make finalize atomic. If this is impossible given the semantics
-     * of the sink, finalize should be idempotent, as it may be called multiple times in the case of
-     * failure/retry or for redundancy.
-     *
-     * <p>Note that the iteration order of the writer results is not guaranteed to be consistent if
-     * finalize is called multiple times.
-     *
-     * @param writerResults an Iterable of results from successful bundle writes.
-     */
-    public abstract void finalize(Iterable<WriteT> writerResults, PipelineOptions options)
-        throws Exception;
-
-    /**
-     * Creates a new {@link Sink.Writer} to write a bundle of the input to the sink.
-     *
-     * <p>The bundle id that the writer will use to uniquely identify its output will be passed to
-     * {@link Writer#openWindowed} or {@link Writer#openUnwindowed}.
-     *
-     * <p>Must not mutate the state of the WriteOperation.
-     */
-    public abstract Writer<T, WriteT> createWriter(PipelineOptions options) throws Exception;
-
-    /**
-     * Returns the Sink that this write operation writes to.
-     */
-    public abstract Sink<T> getSink();
-
-    /**
-     * Returns a coder for the writer result type.
-     */
-    public abstract Coder<WriteT> getWriterResultCoder();
-  }
-
-  /**
-   * A Writer writes a bundle of elements from a PCollection to a sink.
-   * {@link Writer#openWindowed} or {@link Writer#openUnwindowed} is called before writing begins
-   * and {@link Writer#close} is called after all elements in the bundle have been written.
-   * {@link Writer#write} writes an element to the sink.
-   *
-   * <p>Note that any access to static members or methods of a Writer must be thread-safe, as
-   * multiple instances of a Writer may be instantiated in different threads on the same worker.
-   *
-   * <p>See {@link Sink} for more detailed documentation about the process of writing to a Sink.
-   *
-   * @param <T> The type of object to write
-   * @param <WriteT> The writer results type (e.g., the bundle's output filename, as String)
-   */
-  public abstract static class Writer<T, WriteT> {
-    /**
-     * Performs bundle initialization. For example, creates a temporary file for writing or
-     * initializes any state that will be used across calls to {@link Writer#write}.
-     *
-     * <p>The unique id that is given to open should be used to ensure that the writer's output does
-     * not interfere with the output of other Writers, as a bundle may be executed many times for
-     * fault tolerance. See {@link Sink} for more information about bundle ids.
-     *
-     * <p>The window and paneInfo arguments are populated when windowed writes are requested.
-     * shard and numbShards are populated for the case of static sharding. In cases where the
-     * runner is dynamically picking sharding, shard and numShards might both be set to -1.
-     */
-    public abstract void openWindowed(String uId,
-                                      BoundedWindow window,
-                                      PaneInfo paneInfo,
-                                      int shard,
-                                      int numShards) throws Exception;
-
-    /**
-     * Perform bundle initialization for the case where the file is written unwindowed.
-     */
-    public abstract void openUnwindowed(String uId,
-                                        int shard,
-                                        int numShards) throws Exception;
-
-    public abstract void cleanup() throws Exception;
-
-    /**
-     * Called for each value in the bundle.
-     */
-    public abstract void write(T value) throws Exception;
-
-    /**
-     * Finishes writing the bundle. Closes any resources used for writing the bundle.
-     *
-     * <p>Returns a writer result that will be used in the {@link Sink.WriteOperation}'s
-     * finalization. The result should contain some way to identify the output of this bundle (using
-     * the bundle id). {@link WriteOperation#finalize} will use the writer result to identify
-     * successful writes. See {@link Sink} for more information about bundle ids.
-     *
-     * @return the writer result
-     */
-    public abstract WriteT close() throws Exception;
-
-    /**
-     * Returns the write operation this writer belongs to.
-     */
-    public abstract WriteOperation<T, WriteT> getWriteOperation();
-
-
-  }
-}
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/UGIHelper.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/UGIHelper.java
deleted file mode 100644
index fd05a19bea54..000000000000
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/UGIHelper.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.hdfs;
-
-import java.io.IOException;
-import javax.annotation.Nullable;
-import org.apache.hadoop.security.UserGroupInformation;
-
-/**
- * {@link UserGroupInformation} helper methods.
- */
-public class UGIHelper {
-
-  /**
-   * Find the most appropriate UserGroupInformation to use.
-   * @param username the user name, or NULL if none is specified.
-   * @return the most appropriate UserGroupInformation
-   */
-  public static UserGroupInformation getBestUGI(@Nullable String username) throws IOException {
-    return UserGroupInformation.getBestUGI(null, username);
-  }
-
-}
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java
deleted file mode 100644
index ef6556eb481d..000000000000
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/Write.java
+++ /dev/null
@@ -1,588 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.hdfs;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.SerializableCoder;
-import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.hdfs.Sink.WriteOperation;
-import org.apache.beam.sdk.io.hdfs.Sink.Writer;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.ValueProvider;
-import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PDone;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class is deprecated, and only exists currently for HDFSFileSink.
- */
-@Deprecated
-public class Write<T> extends PTransform<PCollection<T>, PDone> {
-  private static final Logger LOG = LoggerFactory.getLogger(Write.class);
-
-  private static final int UNKNOWN_SHARDNUM = -1;
-  private static final int UNKNOWN_NUMSHARDS = -1;
-
-  private final Sink<T> sink;
-  // This allows the number of shards to be dynamically computed based on the input
-  // PCollection.
-  @Nullable
-  private final PTransform<PCollection<T>, PCollectionView<Integer>> computeNumShards;
-  // We don't use a side input for static sharding, as we want this value to be updatable
-  // when a pipeline is updated.
-  @Nullable
-  private final ValueProvider<Integer> numShardsProvider;
-  private boolean windowedWrites;
-
-  /**
-   * Creates a {@link Write} transform that writes to the given {@link Sink}, letting the runner
-   * control how many different shards are produced.
-   */
-  public static <T> Write<T> to(Sink<T> sink) {
-    checkNotNull(sink, "sink");
-    return new Write<>(sink, null /* runner-determined sharding */, null, false);
-  }
-
-  private Write(
-      Sink<T> sink,
-      @Nullable PTransform<PCollection<T>, PCollectionView<Integer>> computeNumShards,
-      @Nullable ValueProvider<Integer> numShardsProvider,
-      boolean windowedWrites) {
-    this.sink = sink;
-    this.computeNumShards = computeNumShards;
-    this.numShardsProvider = numShardsProvider;
-    this.windowedWrites = windowedWrites;
-  }
-
-  @Override
-  public PDone expand(PCollection<T> input) {
-    checkArgument(IsBounded.BOUNDED == input.isBounded() || windowedWrites,
-        "%s can only be applied to an unbounded PCollection if doing windowed writes",
-        Write.class.getSimpleName());
-    return createWrite(input, sink.createWriteOperation());
-  }
-
-  @Override
-  public void validate(PipelineOptions options) {
-    sink.validate(options);
-  }
-
-  @Override
-  public void populateDisplayData(DisplayData.Builder builder) {
-    super.populateDisplayData(builder);
-    builder
-        .add(DisplayData.item("sink", sink.getClass()).withLabel("Write Sink"))
-        .include("sink", sink);
-    if (getSharding() != null) {
-      builder.include("sharding", getSharding());
-    } else if (getNumShards() != null) {
-      String numShards = getNumShards().isAccessible()
-          ? getNumShards().get().toString() : getNumShards().toString();
-      builder.add(DisplayData.item("numShards", numShards)
-          .withLabel("Fixed Number of Shards"));
-    }
-  }
-
-  /**
-   * Returns the {@link Sink} associated with this PTransform.
-   */
-  public Sink<T> getSink() {
-    return sink;
-  }
-
-  /**
-   * Gets the {@link PTransform} that will be used to determine sharding. This can be either a
-   * static number of shards (as following a call to {@link #withNumShards(int)}), dynamic (by
-   * {@link #withSharding(PTransform)}), or runner-determined (by {@link
-   * #withRunnerDeterminedSharding()}.
-   */
-  @Nullable
-  public PTransform<PCollection<T>, PCollectionView<Integer>> getSharding() {
-    return computeNumShards;
-  }
-
-  public ValueProvider<Integer> getNumShards() {
-    return numShardsProvider;
-  }
-
-  /**
-   * Returns a new {@link Write} that will write to the current {@link Sink} using the
-   * specified number of shards.
-   *
-   * <p>This option should be used sparingly as it can hurt performance. See {@link Write} for
-   * more information.
-   *
-   * <p>A value less than or equal to 0 will be equivalent to the default behavior of
-   * runner-determined sharding.
-   */
-  public Write<T> withNumShards(int numShards) {
-    if (numShards > 0) {
-      return withNumShards(StaticValueProvider.of(numShards));
-    }
-    return withRunnerDeterminedSharding();
-  }
-
-  /**
-   * Returns a new {@link Write} that will write to the current {@link Sink} using the
-   * {@link ValueProvider} specified number of shards.
-   *
-   * <p>This option should be used sparingly as it can hurt performance. See {@link Write} for
-   * more information.
-   */
-  public Write<T> withNumShards(ValueProvider<Integer> numShardsProvider) {
-    return new Write<>(sink, null, numShardsProvider, windowedWrites);
-  }
-
-  /**
-   * Returns a new {@link Write} that will write to the current {@link Sink} using the
-   * specified {@link PTransform} to compute the number of shards.
-   *
-   * <p>This option should be used sparingly as it can hurt performance. See {@link Write} for
-   * more information.
-   */
-  public Write<T> withSharding(PTransform<PCollection<T>, PCollectionView<Integer>> sharding) {
-    checkNotNull(
-        sharding, "Cannot provide null sharding. Use withRunnerDeterminedSharding() instead");
-    return new Write<>(sink, sharding, null, windowedWrites);
-  }
-
-  /**
-   * Returns a new {@link Write} that will write to the current {@link Sink} with
-   * runner-determined sharding.
-   */
-  public Write<T> withRunnerDeterminedSharding() {
-    return new Write<>(sink, null, null, windowedWrites);
-  }
-
-  /**
-   * Returns a new {@link Write} that writes preserves windowing on it's input.
-   *
-   * <p>If this option is not specified, windowing and triggering are replaced by
-   * {@link GlobalWindows} and {@link DefaultTrigger}.
-   *
-   * <p>If there is no data for a window, no output shards will be generated for that window.
-   * If a window triggers multiple times, then more than a single output shard might be
-   * generated multiple times; it's up to the sink implementation to keep these output shards
-   * unique.
-   *
-   * <p>This option can only be used if {@link #withNumShards(int)} is also set to a
-   * positive value.
-   */
-  public Write<T> withWindowedWrites() {
-    return new Write<>(sink, computeNumShards, numShardsProvider, true);
-  }
-
-  /**
-   * Writes all the elements in a bundle using a {@link Writer} produced by the
-   * {@link WriteOperation} associated with the {@link Sink}.
-   */
-  private class WriteBundles<WriteT> extends DoFn<T, WriteT> {
-    // Writer that will write the records in this bundle. Lazily
-    // initialized in processElement.
-    private Writer<T, WriteT> writer = null;
-    private BoundedWindow window;
-    private final PCollectionView<WriteOperation<T, WriteT>> writeOperationView;
-
-    WriteBundles(PCollectionView<WriteOperation<T, WriteT>> writeOperationView) {
-      this.writeOperationView = writeOperationView;
-    }
-
-    @ProcessElement
-    public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
-      // Lazily initialize the Writer
-      if (writer == null) {
-        WriteOperation<T, WriteT> writeOperation = c.sideInput(writeOperationView);
-        LOG.info("Opening writer for write operation {}", writeOperation);
-        writer = writeOperation.createWriter(c.getPipelineOptions());
-
-        if (windowedWrites) {
-          writer.openWindowed(UUID.randomUUID().toString(), window, c.pane(), UNKNOWN_SHARDNUM,
-              UNKNOWN_NUMSHARDS);
-        } else {
-          writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, UNKNOWN_NUMSHARDS);
-        }
-        this.window = window;
-        LOG.debug("Done opening writer {} for operation {}", writer, writeOperationView);
-      }
-      try {
-        writer.write(c.element());
-      } catch (Exception e) {
-        // Discard write result and close the write.
-        try {
-          writer.close();
-          // The writer does not need to be reset, as this DoFn cannot be reused.
-        } catch (Exception closeException) {
-          if (closeException instanceof InterruptedException) {
-            // Do not silently ignore interrupted state.
-            Thread.currentThread().interrupt();
-          }
-          // Do not mask the exception that caused the write to fail.
-          e.addSuppressed(closeException);
-        }
-        throw e;
-      }
-    }
-
-    @FinishBundle
-    public void finishBundle(FinishBundleContext c) throws Exception {
-      if (writer != null) {
-        WriteT result = writer.close();
-        c.output(result, window.maxTimestamp(), window);
-        // Reset state in case of reuse.
-        writer = null;
-        window = null;
-      }
-    }
-
-    @Override
-    public void populateDisplayData(DisplayData.Builder builder) {
-      builder.delegate(Write.this);
-    }
-  }
-
-  /**
-   * Like {@link WriteBundles}, but where the elements for each shard have been collected into
-   * a single iterable.
-   *
-   * @see WriteBundles
-   */
-  private class WriteShardedBundles<WriteT> extends DoFn<KV<Integer, Iterable<T>>, WriteT> {
-    private final PCollectionView<WriteOperation<T, WriteT>> writeOperationView;
-    private final PCollectionView<Integer> numShardsView;
-
-    WriteShardedBundles(PCollectionView<WriteOperation<T, WriteT>> writeOperationView,
-                        PCollectionView<Integer> numShardsView) {
-      this.writeOperationView = writeOperationView;
-      this.numShardsView = numShardsView;
-    }
-
-    @ProcessElement
-    public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
-      int numShards = numShardsView != null ? c.sideInput(numShardsView) : getNumShards().get();
-      // In a sharded write, single input element represents one shard. We can open and close
-      // the writer in each call to processElement.
-      WriteOperation<T, WriteT> writeOperation = c.sideInput(writeOperationView);
-      LOG.info("Opening writer for write operation {}", writeOperation);
-      Writer<T, WriteT> writer = writeOperation.createWriter(c.getPipelineOptions());
-      if (windowedWrites) {
-        writer.openWindowed(UUID.randomUUID().toString(), window, c.pane(), c.element().getKey(),
-            numShards);
-      } else {
-        writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, UNKNOWN_NUMSHARDS);
-      }
-      LOG.debug("Done opening writer {} for operation {}", writer, writeOperationView);
-
-      try {
-        try {
-          for (T t : c.element().getValue()) {
-            writer.write(t);
-          }
-        } catch (Exception e) {
-          try {
-            writer.close();
-          } catch (Exception closeException) {
-            if (closeException instanceof InterruptedException) {
-              // Do not silently ignore interrupted state.
-              Thread.currentThread().interrupt();
-            }
-            // Do not mask the exception that caused the write to fail.
-            e.addSuppressed(closeException);
-          }
-          throw e;
-        }
-
-        // Close the writer; if this throws let the error propagate.
-        WriteT result = writer.close();
-        c.output(result);
-      } catch (Exception e) {
-        // If anything goes wrong, make sure to delete the temporary file.
-        writer.cleanup();
-        throw e;
-      }
-    }
-
-    @Override
-    public void populateDisplayData(DisplayData.Builder builder) {
-      builder.delegate(Write.this);
-    }
-  }
-
-  private static class ApplyShardingKey<T> extends DoFn<T, KV<Integer, T>> {
-    private final PCollectionView<Integer> numShardsView;
-    private final ValueProvider<Integer> numShardsProvider;
-    private int shardNumber;
-
-    ApplyShardingKey(PCollectionView<Integer> numShardsView,
-                     ValueProvider<Integer> numShardsProvider) {
-      this.numShardsView = numShardsView;
-      this.numShardsProvider = numShardsProvider;
-      shardNumber = UNKNOWN_SHARDNUM;
-    }
-
-    @ProcessElement
-    public void processElement(ProcessContext context) {
-      int shardCount = 0;
-      if (numShardsView != null) {
-        shardCount = context.sideInput(numShardsView);
-      } else {
-        checkNotNull(numShardsProvider);
-        shardCount = numShardsProvider.get();
-      }
-      checkArgument(
-          shardCount > 0,
-          "Must have a positive number of shards specified for non-runner-determined sharding."
-              + " Got %s",
-          shardCount);
-      if (shardNumber == UNKNOWN_SHARDNUM) {
-        // We want to desynchronize the first record sharding key for each instance of
-        // ApplyShardingKey, so records in a small PCollection will be statistically balanced.
-        shardNumber = ThreadLocalRandom.current().nextInt(shardCount);
-      } else {
-        shardNumber = (shardNumber + 1) % shardCount;
-      }
-      context.output(KV.of(shardNumber, context.element()));
-    }
-  }
-
-  /**
-   * A write is performed as sequence of three {@link ParDo}'s.
-   *
-   * <p>In the first, a do-once ParDo is applied to a singleton PCollection containing the Sink's
-   * {@link WriteOperation}. In this initialization ParDo, {@link WriteOperation#initialize} is
-   * called. The output of this ParDo is a singleton PCollection
-   * containing the WriteOperation.
-   *
-   * <p>This singleton collection containing the WriteOperation is then used as a side input to a
-   * ParDo over the PCollection of elements to write. In this bundle-writing phase,
-   * {@link WriteOperation#createWriter} is called to obtain a {@link Writer}.
-   * {@link Writer#open} and {@link Writer#close} are called in {@link DoFn#startBundle} and
-   * {@link DoFn#finishBundle}, respectively, and {@link Writer#write} method is called for
-   * every element in the bundle. The output of this ParDo is a PCollection of
-   * <i>writer result</i> objects (see {@link Sink} for a description of writer results)-one for
-   * each bundle.
-   *
-   * <p>The final do-once ParDo uses the singleton collection of the WriteOperation as input and
-   * the collection of writer results as a side-input. In this ParDo,
-   * {@link WriteOperation#finalize} is called to finalize the write.
-   *
-   * <p>If the write of any element in the PCollection fails, {@link Writer#close} will be called
-   * before the exception that caused the write to fail is propagated and the write result will be
-   * discarded.
-   *
-   * <p>Since the {@link WriteOperation} is serialized after the initialization ParDo and
-   * deserialized in the bundle-writing and finalization phases, any state change to the
-   * WriteOperation object that occurs during initialization is visible in the latter phases.
-   * However, the WriteOperation is not serialized after the bundle-writing phase.  This is why
-   * implementations should guarantee that {@link WriteOperation#createWriter} does not mutate
-   * WriteOperation).
-   */
-  private <WriteT> PDone createWrite(
-      PCollection<T> input, WriteOperation<T, WriteT> writeOperation) {
-    Pipeline p = input.getPipeline();
-    writeOperation.setWindowedWrites(windowedWrites);
-
-    // A coder to use for the WriteOperation.
-    @SuppressWarnings("unchecked")
-    Coder<WriteOperation<T, WriteT>> operationCoder =
-        (Coder<WriteOperation<T, WriteT>>) SerializableCoder.of(writeOperation.getClass());
-
-    // A singleton collection of the WriteOperation, to be used as input to a ParDo to initialize
-    // the sink.
-    PCollection<WriteOperation<T, WriteT>> operationCollection =
-        p.apply("CreateOperationCollection", Create.of(writeOperation).withCoder(operationCoder));
-
-    // Initialize the resource in a do-once ParDo on the WriteOperation.
-    operationCollection = operationCollection
-        .apply("Initialize", ParDo.of(
-            new DoFn<WriteOperation<T, WriteT>, WriteOperation<T, WriteT>>() {
-              @ProcessElement
-              public void processElement(ProcessContext c) throws Exception {
-                WriteOperation<T, WriteT> writeOperation = c.element();
-                LOG.info("Initializing write operation {}", writeOperation);
-                writeOperation.initialize(c.getPipelineOptions());
-                writeOperation.setWindowedWrites(windowedWrites);
-                LOG.debug("Done initializing write operation {}", writeOperation);
-                // The WriteOperation is also the output of this ParDo, so it can have mutable
-                // state.
-                c.output(writeOperation);
-              }
-            }))
-        .setCoder(operationCoder);
-
-    // Create a view of the WriteOperation to be used as a sideInput to the parallel write phase.
-    final PCollectionView<WriteOperation<T, WriteT>> writeOperationView =
-        operationCollection.apply(View.<WriteOperation<T, WriteT>>asSingleton());
-
-    if (!windowedWrites) {
-      // Re-window the data into the global window and remove any existing triggers.
-      input =
-          input.apply(
-              Window.<T>into(new GlobalWindows())
-                  .triggering(DefaultTrigger.of())
-                  .discardingFiredPanes());
-    }
-
-
-    // Perform the per-bundle writes as a ParDo on the input PCollection (with the WriteOperation
-    // as a side input) and collect the results of the writes in a PCollection.
-    // There is a dependency between this ParDo and the first (the WriteOperation PCollection
-    // as a side input), so this will happen after the initial ParDo.
-    PCollection<WriteT> results;
-    final PCollectionView<Integer> numShardsView;
-    if (computeNumShards == null && numShardsProvider == null) {
-      if (windowedWrites) {
-        throw new IllegalStateException("When doing windowed writes, numShards must be set"
-            + "explicitly to a positive value");
-      }
-      numShardsView = null;
-      results = input
-          .apply("WriteBundles",
-              ParDo.of(new WriteBundles<>(writeOperationView))
-                  .withSideInputs(writeOperationView));
-    } else {
-      if (computeNumShards != null) {
-        numShardsView = input.apply(computeNumShards);
-        results  = input
-            .apply("ApplyShardLabel", ParDo.of(
-                new ApplyShardingKey<T>(numShardsView, null)).withSideInputs(numShardsView))
-            .apply("GroupIntoShards", GroupByKey.<Integer, T>create())
-            .apply("WriteShardedBundles",
-                ParDo.of(new WriteShardedBundles<>(writeOperationView, numShardsView))
-                    .withSideInputs(numShardsView, writeOperationView));
-      } else {
-        numShardsView = null;
-        results = input
-            .apply("ApplyShardLabel", ParDo.of(new ApplyShardingKey<T>(null, numShardsProvider)))
-            .apply("GroupIntoShards", GroupByKey.<Integer, T>create())
-            .apply("WriteShardedBundles",
-                ParDo.of(new WriteShardedBundles<>(writeOperationView, null))
-                    .withSideInputs(writeOperationView));
-      }
-    }
-    results.setCoder(writeOperation.getWriterResultCoder());
-
-    if (windowedWrites) {
-      // When processing streaming windowed writes, results will arrive multiple times. This
-      // means we can't share the below implementation that turns the results into a side input,
-      // as new data arriving into a side input does not trigger the listening DoFn. Instead
-      // we aggregate the result set using a singleton GroupByKey, so the DoFn will be triggered
-      // whenever new data arrives.
-      PCollection<KV<Void, WriteT>> keyedResults =
-          results.apply("AttachSingletonKey", WithKeys.<Void, WriteT>of((Void) null));
-      keyedResults.setCoder(KvCoder.<Void, WriteT>of(VoidCoder.of(), writeOperation
-          .getWriterResultCoder()));
-
-      // Is the continuation trigger sufficient?
-      keyedResults
-          .apply("FinalizeGroupByKey", GroupByKey.<Void, WriteT>create())
-          .apply("Finalize", ParDo.of(new DoFn<KV<Void, Iterable<WriteT>>, Integer>() {
-            @ProcessElement
-            public void processElement(ProcessContext c) throws Exception {
-              WriteOperation<T, WriteT> writeOperation = c.sideInput(writeOperationView);
-              LOG.info("Finalizing write operation {}.", writeOperation);
-              List<WriteT> results = Lists.newArrayList(c.element().getValue());
-              writeOperation.finalize(results, c.getPipelineOptions());
-              LOG.debug("Done finalizing write operation {}", writeOperation);
-            }
-          }).withSideInputs(writeOperationView));
-    } else {
-      final PCollectionView<Iterable<WriteT>> resultsView =
-          results.apply(View.<WriteT>asIterable());
-      ImmutableList.Builder<PCollectionView<?>> sideInputs =
-          ImmutableList.<PCollectionView<?>>builder().add(resultsView);
-      if (numShardsView != null) {
-        sideInputs.add(numShardsView);
-      }
-
-      // Finalize the write in another do-once ParDo on the singleton collection containing the
-      // Writer. The results from the per-bundle writes are given as an Iterable side input.
-      // The WriteOperation's state is the same as after its initialization in the first do-once
-      // ParDo. There is a dependency between this ParDo and the parallel write (the writer
-      // results collection as a side input), so it will happen after the parallel write.
-      // For the non-windowed case, we guarantee that  if no data is written but the user has
-      // set numShards, then all shards will be written out as empty files. For this reason we
-      // use a side input here.
-      operationCollection
-          .apply("Finalize", ParDo.of(new DoFn<WriteOperation<T, WriteT>, Integer>() {
-            @ProcessElement
-            public void processElement(ProcessContext c) throws Exception {
-              WriteOperation<T, WriteT> writeOperation = c.element();
-              LOG.info("Finalizing write operation {}.", writeOperation);
-              List<WriteT> results = Lists.newArrayList(c.sideInput(resultsView));
-              LOG.debug("Side input initialized to finalize write operation {}.", writeOperation);
-
-              // We must always output at least 1 shard, and honor user-specified numShards if
-              // set.
-              int minShardsNeeded;
-              if (numShardsView != null) {
-                minShardsNeeded = c.sideInput(numShardsView);
-              } else if (numShardsProvider != null) {
-                minShardsNeeded = numShardsProvider.get();
-              } else {
-                minShardsNeeded = 1;
-              }
-              int extraShardsNeeded = minShardsNeeded - results.size();
-              if (extraShardsNeeded > 0) {
-                LOG.info(
-                    "Creating {} empty output shards in addition to {} written for a total of "
-                        + " {}.", extraShardsNeeded, results.size(), minShardsNeeded);
-                for (int i = 0; i < extraShardsNeeded; ++i) {
-                  Writer<T, WriteT> writer = writeOperation.createWriter(c.getPipelineOptions());
-                  writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM,
-                      UNKNOWN_NUMSHARDS);
-                  WriteT emptyWrite = writer.close();
-                  results.add(emptyWrite);
-                }
-                LOG.debug("Done creating extra shards.");
-              }
-              writeOperation.finalize(results, c.getPipelineOptions());
-              LOG.debug("Done finalizing write operation {}", writeOperation);
-            }
-          }).withSideInputs(sideInputs.build()));
-    }
-    return PDone.in(input.getPipeline());
-  }
-}
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/package-info.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/package-info.java
index 763b30a7b5ec..32c36cc89a7c 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/package-info.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/package-info.java
@@ -17,6 +17,7 @@
  */
 
 /**
- * Transforms used to read from the Hadoop file system (HDFS).
+ * {@link org.apache.beam.sdk.io.FileSystem} implementation for any Hadoop
+ * {@link org.apache.hadoop.fs.FileSystem}.
  */
 package org.apache.beam.sdk.io.hdfs;
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSinkTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSinkTest.java
deleted file mode 100644
index 9fa6606f04f6..000000000000
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSinkTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.hdfs;
-
-import static org.junit.Assert.assertEquals;
-
-import com.google.common.base.MoreObjects;
-import java.io.File;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.file.FileReader;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.mapred.AvroKey;
-import org.apache.beam.sdk.coders.AvroCoder;
-import org.apache.beam.sdk.coders.DefaultCoder;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.values.KV;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Tests for HDFSFileSinkTest.
- */
-public class HDFSFileSinkTest {
-
-  @Rule
-  public TemporaryFolder tmpFolder = new TemporaryFolder();
-
-  private final String part0 = "part-r-00000";
-  private final String foobar = "foobar";
-
-  private <T> void doWrite(Sink<T> sink,
-                           PipelineOptions options,
-                           Iterable<T> toWrite) throws Exception {
-    Sink.WriteOperation<T, String> writeOperation =
-        (Sink.WriteOperation<T, String>) sink.createWriteOperation();
-    Sink.Writer<T, String> writer = writeOperation.createWriter(options);
-    writer.openUnwindowed(UUID.randomUUID().toString(),  -1, -1);
-    for (T t: toWrite) {
-      writer.write(t);
-    }
-    String writeResult = writer.close();
-    writeOperation.finalize(Collections.singletonList(writeResult), options);
-  }
-
-  @Test
-  public void testWriteSingleRecord() throws Exception {
-    PipelineOptions options = PipelineOptionsFactory.create();
-    File file = tmpFolder.newFolder();
-
-    HDFSFileSink<String, NullWritable, Text> sink =
-        HDFSFileSink.to(
-            file.toString(),
-            SequenceFileOutputFormat.class,
-            NullWritable.class,
-            Text.class,
-            new SerializableFunction<String, KV<NullWritable, Text>>() {
-              @Override
-              public KV<NullWritable, Text> apply(String input) {
-                return KV.of(NullWritable.get(), new Text(input));
-              }
-            });
-
-    doWrite(sink, options, Collections.singletonList(foobar));
-
-    SequenceFile.Reader.Option opts =
-        SequenceFile.Reader.file(new Path(file.toString(), part0));
-    SequenceFile.Reader reader = new SequenceFile.Reader(new Configuration(), opts);
-    assertEquals(NullWritable.class.getName(), reader.getKeyClassName());
-    assertEquals(Text.class.getName(), reader.getValueClassName());
-    NullWritable k = NullWritable.get();
-    Text v = new Text();
-    assertEquals(true, reader.next(k, v));
-    assertEquals(NullWritable.get(), k);
-    assertEquals(new Text(foobar), v);
-  }
-
-  @Test
-  public void testToText() throws Exception {
-    PipelineOptions options = PipelineOptionsFactory.create();
-    File file = tmpFolder.newFolder();
-
-    HDFSFileSink<String, NullWritable, Text> sink = HDFSFileSink.toText(file.toString());
-
-    doWrite(sink, options, Collections.singletonList(foobar));
-
-    List<String> strings = Files.readAllLines(new File(file.toString(), part0).toPath(),
-        Charset.forName("UTF-8"));
-    assertEquals(Collections.singletonList(foobar), strings);
-  }
-
-  @DefaultCoder(AvroCoder.class)
-  static class GenericClass {
-    int intField;
-    String stringField;
-    public GenericClass() {}
-    public GenericClass(int intValue, String stringValue) {
-      this.intField = intValue;
-      this.stringField = stringValue;
-    }
-    @Override
-    public String toString() {
-      return MoreObjects.toStringHelper(getClass())
-          .add("intField", intField)
-          .add("stringField", stringField)
-          .toString();
-    }
-    @Override
-    public int hashCode() {
-      return Objects.hash(intField, stringField);
-    }
-    @Override
-    public boolean equals(Object other) {
-      if (other == null || !(other instanceof GenericClass)) {
-        return false;
-      }
-      GenericClass o = (GenericClass) other;
-      return Objects.equals(intField, o.intField) && Objects.equals(stringField, o.stringField);
-    }
-  }
-
-  @Test
-  public void testToAvro() throws Exception {
-    PipelineOptions options = PipelineOptionsFactory.create();
-    File file = tmpFolder.newFolder();
-
-    HDFSFileSink<GenericClass, AvroKey<GenericClass>, NullWritable> sink = HDFSFileSink.toAvro(
-        file.toString(),
-        AvroCoder.of(GenericClass.class),
-        new Configuration(false));
-
-    doWrite(sink, options, Collections.singletonList(new GenericClass(3, "foobar")));
-
-    GenericDatumReader datumReader = new GenericDatumReader();
-    FileReader<GenericData.Record> reader =
-        DataFileReader.openReader(new File(file.getAbsolutePath(), part0 + ".avro"), datumReader);
-    GenericData.Record next = reader.next(null);
-    assertEquals("foobar", next.get("stringField").toString());
-    assertEquals(3, next.get("intField"));
-  }
-
-}
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSourceTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSourceTest.java
deleted file mode 100644
index a96423901475..000000000000
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HDFSFileSourceTest.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.hdfs;
-
-import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.Source;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.SourceTestUtils;
-import org.apache.beam.sdk.values.KV;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Writer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Tests for HDFSFileSource.
- */
-public class HDFSFileSourceTest {
-
-  private Random random = new Random(0L);
-
-  @Rule
-  public TemporaryFolder tmpFolder = new TemporaryFolder();
-
-  @Test
-  public void testFullyReadSingleFile() throws Exception {
-    PipelineOptions options = PipelineOptionsFactory.create();
-    List<KV<IntWritable, Text>> expectedResults = createRandomRecords(3, 10, 0);
-    File file = createFileWithData("tmp.seq", expectedResults);
-
-    HDFSFileSource<KV<IntWritable, Text>, IntWritable, Text> source =
-            HDFSFileSource.from(
-                    file.toString(), SequenceFileInputFormat.class, IntWritable.class, Text.class);
-
-    assertEquals(file.length(), source.getEstimatedSizeBytes(null));
-
-    assertThat(expectedResults, containsInAnyOrder(readFromSource(source, options).toArray()));
-  }
-
-  @Test
-  public void testFullyReadSingleFileWithSpaces() throws Exception {
-    PipelineOptions options = PipelineOptionsFactory.create();
-    List<KV<IntWritable, Text>> expectedResults = createRandomRecords(3, 10, 0);
-    File file = createFileWithData("tmp data.seq", expectedResults);
-
-    HDFSFileSource<KV<IntWritable, Text>, IntWritable, Text> source =
-            HDFSFileSource.from(
-                    file.toString(), SequenceFileInputFormat.class, IntWritable.class, Text.class);
-
-    assertEquals(file.length(), source.getEstimatedSizeBytes(null));
-
-    assertThat(expectedResults, containsInAnyOrder(readFromSource(source, options).toArray()));
-  }
-
-  @Test
-  public void testFullyReadFilePattern() throws IOException {
-    PipelineOptions options = PipelineOptionsFactory.create();
-    List<KV<IntWritable, Text>> data1 = createRandomRecords(3, 10, 0);
-    File file1 = createFileWithData("file1", data1);
-
-    List<KV<IntWritable, Text>> data2 = createRandomRecords(3, 10, 10);
-    createFileWithData("file2", data2);
-
-    List<KV<IntWritable, Text>> data3 = createRandomRecords(3, 10, 20);
-    createFileWithData("file3", data3);
-
-    List<KV<IntWritable, Text>> data4 = createRandomRecords(3, 10, 30);
-    createFileWithData("otherfile", data4);
-
-    List<KV<IntWritable, Text>> expectedResults = new ArrayList<>();
-    expectedResults.addAll(data1);
-    expectedResults.addAll(data2);
-    expectedResults.addAll(data3);
-
-    HDFSFileSource<KV<IntWritable, Text>, IntWritable, Text> source =
-        HDFSFileSource.from(
-            new File(file1.getParent(), "file*").toString(), SequenceFileInputFormat.class,
-            IntWritable.class, Text.class);
-
-    assertThat(expectedResults, containsInAnyOrder(readFromSource(source, options).toArray()));
-  }
-
-  @Test
-  public void testCloseUnstartedFilePatternReader() throws IOException {
-    PipelineOptions options = PipelineOptionsFactory.create();
-    List<KV<IntWritable, Text>> data1 = createRandomRecords(3, 10, 0);
-    File file1 = createFileWithData("file1", data1);
-
-    List<KV<IntWritable, Text>> data2 = createRandomRecords(3, 10, 10);
-    createFileWithData("file2", data2);
-
-    List<KV<IntWritable, Text>> data3 = createRandomRecords(3, 10, 20);
-    createFileWithData("file3", data3);
-
-    List<KV<IntWritable, Text>> data4 = createRandomRecords(3, 10, 30);
-    createFileWithData("otherfile", data4);
-
-    HDFSFileSource<KV<IntWritable, Text>, IntWritable, Text> source =
-        HDFSFileSource.from(
-            new File(file1.getParent(), "file*").toString(),
-            SequenceFileInputFormat.class, IntWritable.class, Text.class);
-    Source.Reader<KV<IntWritable, Text>> reader = source.createReader(options);
-
-    // Closing an unstarted FilePatternReader should not throw an exception.
-    try {
-      reader.close();
-    } catch (Exception e) {
-      fail("Closing an unstarted FilePatternReader should not throw an exception");
-    }
-  }
-
-  @Test
-  public void testSplits() throws Exception {
-    PipelineOptions options = PipelineOptionsFactory.create();
-
-    List<KV<IntWritable, Text>> expectedResults = createRandomRecords(3, 10000, 0);
-    File file = createFileWithData("tmp.seq", expectedResults);
-
-    HDFSFileSource<KV<IntWritable, Text>, IntWritable, Text> source =
-        HDFSFileSource.from(
-            file.toString(), SequenceFileInputFormat.class, IntWritable.class, Text.class);
-
-    // Assert that the source produces the expected records
-    assertEquals(expectedResults, readFromSource(source, options));
-
-    // Split with a small bundle size (has to be at least size of sync interval)
-    List<? extends BoundedSource<KV<IntWritable, Text>>> splits = source
-        .split(SequenceFile.SYNC_INTERVAL, options);
-    assertTrue(splits.size() > 2);
-    SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
-    int nonEmptySplits = 0;
-    for (BoundedSource<KV<IntWritable, Text>> subSource : splits) {
-      if (readFromSource(subSource, options).size() > 0) {
-        nonEmptySplits += 1;
-      }
-    }
-    assertTrue(nonEmptySplits > 2);
-  }
-
-  @Test
-  public void testSplitEstimatedSize() throws Exception {
-    PipelineOptions options = PipelineOptionsFactory.create();
-
-    List<KV<IntWritable, Text>> expectedResults = createRandomRecords(3, 10000, 0);
-    File file = createFileWithData("tmp.avro", expectedResults);
-
-    HDFSFileSource<KV<IntWritable, Text>, IntWritable, Text> source =
-        HDFSFileSource.from(file.toString(), SequenceFileInputFormat.class,
-            IntWritable.class, Text.class);
-
-    long originalSize = source.getEstimatedSizeBytes(options);
-    long splitTotalSize = 0;
-    List<? extends BoundedSource<KV<IntWritable, Text>>> splits = source.split(
-        SequenceFile.SYNC_INTERVAL, options
-    );
-    for (BoundedSource<KV<IntWritable, Text>> splitSource : splits) {
-      splitTotalSize += splitSource.getEstimatedSizeBytes(options);
-    }
-    // Assert that the estimated size of the whole is the sum of its parts
-    assertEquals(originalSize, splitTotalSize);
-  }
-
-  private File createFileWithData(String filename, List<KV<IntWritable, Text>> records)
-      throws IOException {
-    File tmpFile = tmpFolder.newFile(filename);
-    try (Writer writer = SequenceFile.createWriter(new Configuration(),
-        Writer.keyClass(IntWritable.class), Writer.valueClass(Text.class),
-        Writer.file(new Path(tmpFile.toURI())))) {
-
-      for (KV<IntWritable, Text> record : records) {
-        writer.append(record.getKey(), record.getValue());
-      }
-    }
-    return tmpFile;
-  }
-
-  private List<KV<IntWritable, Text>> createRandomRecords(int dataItemLength,
-                                                          int numItems, int offset) {
-    List<KV<IntWritable, Text>> records = new ArrayList<>();
-    for (int i = 0; i < numItems; i++) {
-      IntWritable key = new IntWritable(i + offset);
-      Text value = new Text(createRandomString(dataItemLength));
-      records.add(KV.of(key, value));
-    }
-    return records;
-  }
-
-  private String createRandomString(int length) {
-    char[] chars = "abcdefghijklmnopqrstuvwxyz".toCharArray();
-    StringBuilder builder = new StringBuilder();
-    for (int i = 0; i < length; i++) {
-      builder.append(chars[random.nextInt(chars.length)]);
-    }
-    return builder.toString();
-  }
-
-}

From c2f47515693ef8046b3a6bb8a892af7b5a8df59b Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 4 May 2017 13:46:24 -0700
Subject: [PATCH 070/387] Run Optimize Imports on the Repository

---
 .../test/java/org/apache/beam/examples/WindowedWordCountIT.java  | 1 -
 .../examples/complete/game/utils/WriteWindowedToBigQuery.java    | 1 -
 .../beam/runners/apex/translation/TransformTranslator.java       | 1 -
 .../apache/beam/runners/dataflow/util/DataflowTemplateJob.java   | 1 -
 .../java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java     | 1 -
 5 files changed, 5 deletions(-)

diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index b5eddb5d17e7..f7e35c023172 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -30,7 +30,6 @@
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.ThreadLocalRandom;
-
 import org.apache.beam.examples.common.ExampleUtils;
 import org.apache.beam.examples.common.WriteOneFilePerWindow.PerWindowFiles;
 import org.apache.beam.sdk.PipelineResult;
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java
index deb9db2c0fd5..db3319c34985 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteWindowedToBigQuery.java
@@ -19,7 +19,6 @@
 
 import com.google.api.services.bigquery.model.TableRow;
 import java.util.Map;
-
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java
index 49ff49b4dbf3..c924b2ee53ef 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TransformTranslator.java
@@ -18,7 +18,6 @@
 
 package org.apache.beam.runners.apex.translation;
 
-
 import java.io.Serializable;
 import org.apache.beam.sdk.transforms.PTransform;
 
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java
index 2937184090a8..b4644620d038 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java
@@ -19,7 +19,6 @@
 import com.google.common.annotations.VisibleForTesting;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.dataflow.DataflowPipelineJob;
-import org.apache.beam.sdk.PipelineResult.State;
 import org.joda.time.Duration;
 
 /**
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index fbbf862d1e84..6e410c7b6c28 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -38,7 +38,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;

From 2ac2a34b90b8d03ef71b01a75c14a9b16fa482b9 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 4 May 2017 14:07:36 -0700
Subject: [PATCH 071/387] Move AppliedPTransform into the Runners package

This is a read-only view of the TransformHierarchy.Node, and is intended
for use by Runner authors.
---
 .../src/main/java/org/apache/beam/runners/apex/ApexRunner.java | 2 +-
 .../beam/runners/apex/translation/TranslationContext.java      | 2 +-
 .../runners/core/construction/DeduplicatedFlattenFactory.java  | 2 +-
 .../runners/core/construction/EmptyFlattenAsCreateFactory.java | 2 +-
 .../beam/runners/core/construction/PTransformMatchers.java     | 2 +-
 .../beam/runners/core/construction/PTransformReplacements.java | 2 +-
 .../org/apache/beam/runners/core/construction/PTransforms.java | 2 +-
 .../apache/beam/runners/core/construction/PrimitiveCreate.java | 2 +-
 .../apache/beam/runners/core/construction/SdkComponents.java   | 2 +-
 .../runners/core/construction/UnsupportedOverrideFactory.java  | 2 +-
 .../core/construction/EmptyFlattenAsCreateFactoryTest.java     | 2 +-
 .../beam/runners/core/construction/PTransformMatchersTest.java | 2 +-
 .../runners/core/construction/PTransformReplacementsTest.java  | 2 +-
 .../apache/beam/runners/core/construction/PTransformsTest.java | 2 +-
 .../beam/runners/core/construction/SdkComponentsTest.java      | 2 +-
 .../construction/SingleInputOutputOverrideFactoryTest.java     | 2 +-
 .../beam/runners/direct/BoundedReadEvaluatorFactory.java       | 2 +-
 .../java/org/apache/beam/runners/direct/CommittedResult.java   | 2 +-
 .../org/apache/beam/runners/direct/CompletionCallback.java     | 2 +-
 .../direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java     | 2 +-
 .../main/java/org/apache/beam/runners/direct/DirectGraph.java  | 2 +-
 .../org/apache/beam/runners/direct/DirectGraphVisitor.java     | 2 +-
 .../beam/runners/direct/DirectGroupByKeyOverrideFactory.java   | 2 +-
 .../org/apache/beam/runners/direct/EmptyInputProvider.java     | 2 +-
 .../java/org/apache/beam/runners/direct/EvaluationContext.java | 2 +-
 .../beam/runners/direct/ExecutorServiceParallelExecutor.java   | 2 +-
 .../apache/beam/runners/direct/FlattenEvaluatorFactory.java    | 2 +-
 .../beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java | 2 +-
 .../beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java    | 2 +-
 .../beam/runners/direct/ImmutabilityEnforcementFactory.java    | 2 +-
 .../java/org/apache/beam/runners/direct/ModelEnforcement.java  | 2 +-
 .../apache/beam/runners/direct/ModelEnforcementFactory.java    | 2 +-
 .../java/org/apache/beam/runners/direct/ParDoEvaluator.java    | 2 +-
 .../org/apache/beam/runners/direct/ParDoEvaluatorFactory.java  | 2 +-
 .../apache/beam/runners/direct/ParDoMultiOverrideFactory.java  | 2 +-
 .../beam/runners/direct/PassthroughTransformEvaluator.java     | 2 +-
 .../java/org/apache/beam/runners/direct/PipelineExecutor.java  | 2 +-
 .../java/org/apache/beam/runners/direct/RootInputProvider.java | 2 +-
 .../org/apache/beam/runners/direct/RootProviderRegistry.java   | 2 +-
 .../direct/SplittableProcessElementsEvaluatorFactory.java      | 2 +-
 .../beam/runners/direct/StatefulParDoEvaluatorFactory.java     | 2 +-
 .../main/java/org/apache/beam/runners/direct/StepAndKey.java   | 2 +-
 .../org/apache/beam/runners/direct/StepTransformResult.java    | 2 +-
 .../apache/beam/runners/direct/TestStreamEvaluatorFactory.java | 2 +-
 .../apache/beam/runners/direct/TransformEvaluatorFactory.java  | 2 +-
 .../apache/beam/runners/direct/TransformEvaluatorRegistry.java | 2 +-
 .../java/org/apache/beam/runners/direct/TransformExecutor.java | 2 +-
 .../java/org/apache/beam/runners/direct/TransformResult.java   | 2 +-
 .../beam/runners/direct/UnboundedReadEvaluatorFactory.java     | 2 +-
 .../org/apache/beam/runners/direct/ViewEvaluatorFactory.java   | 2 +-
 .../org/apache/beam/runners/direct/ViewOverrideFactory.java    | 2 +-
 .../apache/beam/runners/direct/WatermarkCallbackExecutor.java  | 2 +-
 .../java/org/apache/beam/runners/direct/WatermarkManager.java  | 2 +-
 .../org/apache/beam/runners/direct/WindowEvaluatorFactory.java | 2 +-
 .../apache/beam/runners/direct/WriteWithShardingFactory.java   | 2 +-
 .../beam/runners/direct/BoundedReadEvaluatorFactoryTest.java   | 2 +-
 .../org/apache/beam/runners/direct/CommittedResultTest.java    | 2 +-
 .../org/apache/beam/runners/direct/DirectGraphVisitorTest.java | 2 +-
 .../test/java/org/apache/beam/runners/direct/DirectGraphs.java | 2 +-
 .../runners/direct/DirectGroupByKeyOverrideFactoryTest.java    | 2 +-
 .../org/apache/beam/runners/direct/EvaluationContextTest.java  | 2 +-
 .../beam/runners/direct/FlattenEvaluatorFactoryTest.java       | 2 +-
 .../runners/direct/ImmutabilityEnforcementFactoryTest.java     | 2 +-
 .../org/apache/beam/runners/direct/ParDoEvaluatorTest.java     | 2 +-
 .../beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java | 2 +-
 .../apache/beam/runners/direct/StepTransformResultTest.java    | 2 +-
 .../beam/runners/direct/TestStreamEvaluatorFactoryTest.java    | 2 +-
 .../org/apache/beam/runners/direct/TransformExecutorTest.java  | 2 +-
 .../beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java | 2 +-
 .../apache/beam/runners/direct/ViewEvaluatorFactoryTest.java   | 2 +-
 .../apache/beam/runners/direct/ViewOverrideFactoryTest.java    | 2 +-
 .../beam/runners/direct/WatermarkCallbackExecutorTest.java     | 2 +-
 .../org/apache/beam/runners/direct/WatermarkManagerTest.java   | 2 +-
 .../beam/runners/direct/WriteWithShardingFactoryTest.java      | 2 +-
 .../beam/runners/flink/FlinkBatchTranslationContext.java       | 2 +-
 .../beam/runners/flink/FlinkStreamingPipelineTranslator.java   | 2 +-
 .../beam/runners/flink/FlinkStreamingTranslationContext.java   | 2 +-
 .../beam/runners/dataflow/BatchStatefulParDoOverrides.java     | 2 +-
 .../org/apache/beam/runners/dataflow/BatchViewOverrides.java   | 2 +-
 .../org/apache/beam/runners/dataflow/DataflowPipelineJob.java  | 2 +-
 .../beam/runners/dataflow/DataflowPipelineTranslator.java      | 2 +-
 .../java/org/apache/beam/runners/dataflow/DataflowRunner.java  | 2 +-
 .../beam/runners/dataflow/PrimitiveParDoSingleFactory.java     | 2 +-
 .../apache/beam/runners/dataflow/ReshuffleOverrideFactory.java | 2 +-
 .../apache/beam/runners/dataflow/StreamingViewOverrides.java   | 2 +-
 .../org/apache/beam/runners/dataflow/TransformTranslator.java  | 2 +-
 .../apache/beam/runners/dataflow/DataflowPipelineJobTest.java  | 2 +-
 .../beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java | 2 +-
 .../main/java/org/apache/beam/runners/spark/SparkRunner.java   | 2 +-
 .../java/org/apache/beam/runners/spark/TestSparkRunner.java    | 2 +-
 .../beam/runners/spark/translation/EvaluationContext.java      | 2 +-
 .../spark/translation/streaming/TrackStreamingSourcesTest.java | 2 +-
 sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java | 2 +-
 .../beam/sdk/{transforms => runners}/AppliedPTransform.java    | 3 ++-
 .../java/org/apache/beam/sdk/runners/PTransformMatcher.java    | 1 -
 .../org/apache/beam/sdk/runners/PTransformOverrideFactory.java | 1 -
 .../java/org/apache/beam/sdk/runners/TransformHierarchy.java   | 1 -
 .../core/src/test/java/org/apache/beam/sdk/PipelineTest.java   | 2 +-
 98 files changed, 96 insertions(+), 98 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{transforms => runners}/AppliedPTransform.java (96%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index e1828c3217c6..ede41bc4c351 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -48,9 +48,9 @@
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
 import org.apache.beam.sdk.transforms.Create;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index c78028ec5720..9c20449d07e0 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -35,8 +35,8 @@
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java
index 13e7593057d2..bd65843b1632 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java
@@ -22,8 +22,8 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.Flatten.PCollections;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java
index a6982d4801f2..f5be89b10ab0 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java
@@ -22,8 +22,8 @@
 
 import java.util.Map;
 import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.Flatten.PCollections;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
index b2bd7d93ce9a..bfe24a02ab63 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
@@ -23,8 +23,8 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformMatcher;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java
index 72a3425bada8..706a9564640b 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java
@@ -22,7 +22,7 @@
 
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValue;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
index 7ec0863860b6..6d2c6b6fe053 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
@@ -26,7 +26,7 @@
 import java.util.Map;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValue;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
index a791a0e20a03..f43d23b88b8b 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java
@@ -19,8 +19,8 @@
 package org.apache.beam.runners.core.construction;
 
 import java.util.Map;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Create.Values;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index 3e7807221e49..2de8237c8f36 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -31,7 +31,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnsupportedOverrideFactory.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnsupportedOverrideFactory.java
index efafa33b3513..3efbd7c59b06 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnsupportedOverrideFactory.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnsupportedOverrideFactory.java
@@ -19,8 +19,8 @@
 package org.apache.beam.runners.core.construction;
 
 import java.util.Map;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactoryTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactoryTest.java
index c388878bf97d..0345fef730e6 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactoryTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactoryTest.java
@@ -24,12 +24,12 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
 import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.Flatten.PCollections;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index 6271234db021..9ae236be0737 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -36,6 +36,7 @@
 import org.apache.beam.sdk.io.WriteFiles;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformMatcher;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
@@ -45,7 +46,6 @@
 import org.apache.beam.sdk.state.TimerSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java
index c6c50f067d1e..1318feb17ea4 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java
@@ -24,8 +24,8 @@
 import com.google.common.collect.ImmutableMap;
 import java.util.Collections;
 import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
index 4ef70c08e56c..41255448dee7 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
@@ -38,8 +38,8 @@
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
index f605e015cc09..82840d670973 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
@@ -35,8 +35,8 @@
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
 import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SingleInputOutputOverrideFactoryTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SingleInputOutputOverrideFactoryTest.java
index acca5cd28b84..80bb0a7a6f2c 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SingleInputOutputOverrideFactoryTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SingleInputOutputOverrideFactoryTest.java
@@ -22,9 +22,9 @@
 
 import java.io.Serializable;
 import java.util.Map;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.SimpleFunction;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
index 26f985129f2b..99a0fca2e72d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
@@ -37,7 +37,7 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.Read.Bounded;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.UserCodeException;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
index 99abdd31a2df..8c45449491b5 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
@@ -21,7 +21,7 @@
 import com.google.auto.value.AutoValue;
 import java.util.Set;
 import javax.annotation.Nullable;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
index 7b5ef4b530a1..0af22c83707e 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.runners.direct;
 
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 
 /**
  * A callback for completing a bundle of input.
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
index 112024341aa9..64eecc852afd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
@@ -21,7 +21,7 @@
 import org.apache.beam.runners.core.SplittableParDo.GBKIntoKeyedWorkItems;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
index e163d83f3200..83b214aca89a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
@@ -23,7 +23,7 @@
 import java.util.Map;
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index c3421363bbe8..1ee8ceb9a7e5 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -27,8 +27,8 @@
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PInput;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
index 4eb036300c4e..c2eb5e72b842 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
@@ -19,8 +19,8 @@
 
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
index 396cdeed2460..c36879a5f7b6 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
@@ -19,7 +19,7 @@
 
 import java.util.Collection;
 import java.util.Collections;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
index 7b6461103477..362ff91b893c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -40,7 +40,7 @@
 import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
 import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.Trigger;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index 4da62d583281..02fb11aea315 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -50,7 +50,7 @@
 import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
index 341ea4b4d2bf..1b169b8f8510 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
@@ -18,7 +18,7 @@
 package org.apache.beam.runners.direct;
 
 import com.google.common.collect.Iterables;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.Flatten.PCollections;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 83548589c1eb..41c797f840b2 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -42,7 +42,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java
index 1ea8e76051e9..08623efca2dd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java
@@ -35,7 +35,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
index 8880af9eaa87..4a67fb4a6bfb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
@@ -21,7 +21,7 @@
 import java.util.Map;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.util.IllegalMutationException;
 import org.apache.beam.sdk.util.MutationDetector;
 import org.apache.beam.sdk.util.MutationDetectors;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
index d2e942470c90..da5127fdc5a8 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
@@ -18,7 +18,7 @@
 package org.apache.beam.runners.direct;
 
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java
index 30f1d20e9ece..2317fc03a030 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.runners.direct;
 
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 
 /**
  * Creates {@link ModelEnforcement} instances for an {@link AppliedPTransform} on an input
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
index 4cfd16fec901..28fc68d293cb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
@@ -31,7 +31,7 @@
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.UserCodeException;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 39595d86afa7..74470bfb8b8d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -25,7 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
index c72bf61e2c5f..89903da1ce00 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
@@ -29,8 +29,8 @@
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
index c57932c80f00..317749373a9b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.runners.direct;
 
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.util.WindowedValue;
 
 class PassthroughTransformEvaluator<InputT> implements TransformEvaluator<InputT> {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java
index 07212c709637..d954fa206eb0 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java
@@ -20,7 +20,7 @@
 import java.util.Collection;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.joda.time.Duration;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
index 88e0769d58f4..ce6951805117 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
@@ -20,7 +20,7 @@
 
 import java.util.Collection;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PInput;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
index b06a41c3cd32..4b0c06d3611f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
@@ -23,7 +23,7 @@
 import java.util.Collection;
 import java.util.Map;
 import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Flatten.PCollections;
 import org.apache.beam.sdk.transforms.PTransform;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index 44f2e853f97c..27972335d703 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -35,7 +35,7 @@
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.TimerInternalsFactory;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 02472b3e69f6..985c3be4e9e9 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -39,8 +39,8 @@
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.ParDoMultiOverrideFactory.StatefulParDo;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.state.StateSpec;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java
index e18b2ac9b6aa..5440dde25e45 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java
@@ -19,7 +19,7 @@
 
 import com.google.common.base.MoreObjects;
 import java.util.Objects;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 
 /**
  * A (Step, Key) pair. This is useful as a map key or cache key for things that are available
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
index 7e5f82433136..c7f78472f627 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
@@ -26,7 +26,7 @@
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.sdk.metrics.MetricUpdates;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.joda.time.Instant;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
index ea989d754668..8b21d5affd60 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
@@ -29,6 +29,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.testing.TestStream.ElementEvent;
@@ -36,7 +37,6 @@
 import org.apache.beam.sdk.testing.TestStream.EventType;
 import org.apache.beam.sdk.testing.TestStream.ProcessingTimeEvent;
 import org.apache.beam.sdk.testing.TestStream.WatermarkEvent;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
index c1873592d18f..72e126062fbe 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
@@ -20,7 +20,7 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
index a00253a51a44..d0e622d7f934 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
@@ -31,7 +31,7 @@
 import org.apache.beam.runners.direct.ParDoMultiOverrideFactory.StatefulParDo;
 import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
 import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Flatten.PCollections;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
index 26c4f5cf7bce..8e1515bb1901 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
@@ -24,7 +24,7 @@
 import org.apache.beam.sdk.metrics.MetricUpdates;
 import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.util.WindowedValue;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
index 0b0790ef0551..8d7aeda72fcf 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
@@ -22,7 +22,7 @@
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.sdk.metrics.MetricUpdates;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
index 922a681248cd..cba826ccc98c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
@@ -35,7 +35,7 @@
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
 import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
index f4648e9e5618..057f4a1836ca 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
@@ -23,7 +23,7 @@
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
 import org.apache.beam.runners.direct.StepTransformResult.Builder;
 import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
index d4fd18fa1414..b3bbac827fd3 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
@@ -24,8 +24,8 @@
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.Values;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java
index ece5a56732fd..bbe503463a53 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java
@@ -24,7 +24,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.Executor;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index 21ba734c017c..4f1b8319dc2d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -55,8 +55,8 @@
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.state.TimeDomain;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
index 30d507be2d02..f4228d98b341 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
@@ -20,7 +20,7 @@
 import com.google.common.collect.Iterables;
 import java.util.Collection;
 import javax.annotation.Nullable;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.Window;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
index 24462e59eee2..65a5a19382c2 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
@@ -27,8 +27,8 @@
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.sdk.io.WriteFiles;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
index df7c18e41f89..6180d2994a1a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
@@ -50,9 +50,9 @@
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.SourceTestUtils;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
index 3a7fdf3a913b..cf19dc20e7e6 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
@@ -27,8 +27,8 @@
 import java.util.EnumSet;
 import java.util.List;
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
index 2afb6c39caec..7f46a0e4c9c3 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
@@ -29,8 +29,8 @@
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
index 73ada196104f..2f048fa7cab2 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
@@ -18,7 +18,7 @@
 package org.apache.beam.runners.direct;
 
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.values.PValue;
 
 /** Test utilities for the {@link DirectRunner}. */
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactoryTest.java
index 28fef4c7e5d5..60ced7ec5885 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactoryTest.java
@@ -23,9 +23,9 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 077bb6865cfc..72b1bbcba188 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -44,10 +44,10 @@
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.state.BagState;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.WithKeys;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
index 7dc01e6332a2..615016c9d4af 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
@@ -25,8 +25,8 @@
 
 import com.google.common.collect.Iterables;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
index 1cd578601623..c0919b9509fc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
@@ -19,8 +19,8 @@
 
 import java.io.Serializable;
 import java.util.Collections;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 88fd5d2f159d..286e44d1be04 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -33,8 +33,8 @@
 import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
index f1ba57cf8c49..eb54d5c818dc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
@@ -45,11 +45,11 @@
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StepTransformResultTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StepTransformResultTest.java
index 45ae678694ad..1ea854c538a2 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StepTransformResultTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StepTransformResultTest.java
@@ -24,8 +24,8 @@
 import static org.junit.Assert.assertThat;
 
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.values.PCollection;
 import org.hamcrest.Matchers;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java
index d1712e0a1630..30c2bde974dc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java
@@ -30,9 +30,9 @@
 import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.TestClock;
 import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.TestStreamIndex;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestStream;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
index 1f95327dd65c..dc0ef7ca7828 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
@@ -36,8 +36,8 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index 68129ad09a90..ceb078be3058 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -57,9 +57,9 @@
 import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.SourceTestUtils;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
index d8869b2ca25d..419698e29798 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
@@ -27,8 +27,8 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.Values;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
index eda00a785e0c..024e15c4c536 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
@@ -30,11 +30,11 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
index f4a53da28368..b66734600ee9 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
@@ -23,8 +23,8 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
index e1e6ab59cdd4..9528ac93285e 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
@@ -43,9 +43,9 @@
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Filter;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
index a2b0c5ce7733..5a2a328beab0 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
@@ -48,8 +48,8 @@
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFnTester;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index e96d1c5ec5e4..0439119dfc40 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -23,7 +23,7 @@
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
index 42d75cf9a261..35d1bcd19f68 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
@@ -28,10 +28,10 @@
 import org.apache.beam.runners.core.construction.UnconsumedReads;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
index 45ee14d25a0d..ea5f6b3162af 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
@@ -25,7 +25,7 @@
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
index 13ae9ee03b8d..4d9a57fbf977 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java
@@ -26,8 +26,8 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index debaf59ef93e..ef2bfed39e7c 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -57,7 +57,7 @@
 import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
 import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index aef31554c7bf..23084ed50498 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -45,7 +45,7 @@
 import org.apache.beam.runners.dataflow.util.MonitoringUtil;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 92f9c9823410..840bda846ed6 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -75,8 +75,8 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.StreamingOptions;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 57da61b905b7..4198baea7517 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -99,12 +99,12 @@
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Combine.GroupedValues;
 import org.apache.beam.sdk.transforms.DoFn;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
index 2e50cb5f7b4b..8611d3cad945 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java
@@ -23,8 +23,8 @@
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.DisplayData;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
index 5814efd0ce47..cd9378c60822 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReshuffleOverrideFactory.java
@@ -20,8 +20,8 @@
 
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
index eb385de7fa3f..6c385d74085b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
@@ -26,7 +26,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java
index 52b3a31b1f4c..23949bd23208 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TransformTranslator.java
@@ -23,7 +23,7 @@
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
 import org.apache.beam.runners.dataflow.util.OutputReference;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PInput;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
index 237493a3247a..f868a174ba65 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
@@ -55,9 +55,9 @@
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.ExpectedLogs;
 import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.values.PInput;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java
index e320036d62dc..4390e7345818 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java
@@ -27,9 +27,9 @@
 import java.util.List;
 import org.apache.beam.runners.dataflow.PrimitiveParDoSingleFactory.ParDoSingle;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 1a0c04234da2..16d9fdcfbbe7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -46,9 +46,9 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index a6851c49ded5..ada7fb58fd7b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -42,10 +42,10 @@
 import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PBegin;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 5d77e9162e7d..8102926f6daa 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -31,7 +31,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index 3dcab26e946b..33a636ad65f8 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -33,8 +33,8 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index f7c3f24c46bd..1769c0522c9c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -33,6 +33,7 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
@@ -40,7 +41,6 @@
 import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.UserCodeException;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AppliedPTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AppliedPTransform.java
similarity index 96%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AppliedPTransform.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AppliedPTransform.java
index 4e049a590d7e..7a3532e8117a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AppliedPTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AppliedPTransform.java
@@ -15,12 +15,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.transforms;
+package org.apache.beam.sdk.runners;
 
 import com.google.auto.value.AutoValue;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java
index 6378ecc82f6e..350d0d98f7a6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java
@@ -21,7 +21,6 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
index a28f303688d6..5f012dedca06 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
@@ -24,7 +24,6 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 703aeb684346..fac558bc8432 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -38,7 +38,6 @@
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
index fda64b5a85e1..d6b527ccf6ee 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
@@ -40,6 +40,7 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptions.CheckEnabled;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformMatcher;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
@@ -51,7 +52,6 @@
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.ValidatesRunner;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Flatten;

From 72be5c71561bf552c25e2de2b0d21aa374b17ec0 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 5 May 2017 09:20:10 -0700
Subject: [PATCH 072/387] Convert Coder into an Abstract Static Class

---
 runners/google-cloud-dataflow-java/pom.xml    |  2 +-
 .../org/apache/beam/sdk/coders/Coder.java     | 36 +++++++++----------
 .../beam/sdk/coders/CoderFactories.java       | 35 +++++++++---------
 .../beam/sdk/coders/StructuredCoder.java      |  2 +-
 .../DoFnSignaturesSplittableDoFnTest.java     |  4 +--
 5 files changed, 40 insertions(+), 39 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index e21b6de0f784..c7142fe024b2 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170504-2</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170505-wd-2914</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index c923719eb025..169e448623c7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -57,10 +57,10 @@
  *
  * @param <T> the type of the values being transcoded
  */
-public interface Coder<T> extends Serializable {
+public abstract class Coder<T> implements Serializable {
   /** The context in which encoding or decoding is being done. */
   @Deprecated
-  class Context {
+  public static class Context {
     /**
      * The outer context: the value being encoded or decoded takes
      * up the remainder of the record/stream contents.
@@ -118,7 +118,7 @@ public String toString() {
    * for some reason
    * @throws CoderException if the value could not be encoded for some reason
    */
-  void encode(T value, OutputStream outStream)
+  public abstract void encode(T value, OutputStream outStream)
       throws CoderException, IOException;
 
   /**
@@ -130,7 +130,7 @@ void encode(T value, OutputStream outStream)
    * @throws CoderException if the value could not be encoded for some reason
    */
   @Deprecated
-  void encodeOuter(T value, OutputStream outStream)
+  public abstract void encodeOuter(T value, OutputStream outStream)
       throws CoderException, IOException;
 
   /**
@@ -142,7 +142,7 @@ void encodeOuter(T value, OutputStream outStream)
    * @throws CoderException if the value could not be encoded for some reason
    */
   @Deprecated
-  void encode(T value, OutputStream outStream, Context context)
+  public abstract void encode(T value, OutputStream outStream, Context context)
       throws CoderException, IOException;
 
   /**
@@ -153,7 +153,7 @@ void encode(T value, OutputStream outStream, Context context)
    * for some reason
    * @throws CoderException if the value could not be decoded for some reason
    */
-  T decode(InputStream inStream) throws CoderException, IOException;
+  public abstract T decode(InputStream inStream) throws CoderException, IOException;
 
   /**
    * Decodes a value of type {@code T} from the given input stream in
@@ -164,7 +164,7 @@ void encode(T value, OutputStream outStream, Context context)
    * @throws CoderException if the value could not be decoded for some reason
    */
   @Deprecated
-  T decodeOuter(InputStream inStream) throws CoderException, IOException;
+  public abstract T decodeOuter(InputStream inStream) throws CoderException, IOException;
 
   /**
    * Decodes a value of type {@code T} from the given input stream in
@@ -175,7 +175,7 @@ void encode(T value, OutputStream outStream, Context context)
    * @throws CoderException if the value could not be decoded for some reason
    */
   @Deprecated
-  T decode(InputStream inStream, Context context)
+  public abstract T decode(InputStream inStream, Context context)
       throws CoderException, IOException;
 
   /**
@@ -184,7 +184,7 @@ T decode(InputStream inStream, Context context)
    * returns {@code null} if this cannot be done or this is not a
    * parameterized type.
    */
-  List<? extends Coder<?>> getCoderArguments();
+  public abstract List<? extends Coder<?>> getCoderArguments();
 
   /**
    * Throw {@link NonDeterministicException} if the coding is not deterministic.
@@ -202,7 +202,7 @@ T decode(InputStream inStream, Context context)
    *
    * @throws Coder.NonDeterministicException if this coder is not deterministic.
    */
-  void verifyDeterministic() throws Coder.NonDeterministicException;
+  public abstract void verifyDeterministic() throws Coder.NonDeterministicException;
 
   /**
    * Returns {@code true} if this {@link Coder} is injective with respect to {@link Objects#equals}.
@@ -214,7 +214,7 @@ T decode(InputStream inStream, Context context)
    * whenever {@code equals()} compares object identity, rather than performing a
    * semantic/structural comparison.
    */
-  boolean consistentWithEquals();
+  public abstract boolean consistentWithEquals();
 
   /**
    * Returns an object with an {@code Object.equals()} method that represents structural equality
@@ -234,7 +234,7 @@ T decode(InputStream inStream, Context context)
    *
    * <p>See also {@link #consistentWithEquals()}.
    */
-  Object structuralValue(T value);
+  public abstract Object structuralValue(T value);
 
   /**
    * Returns whether {@link #registerByteSizeObserver} cheap enough to
@@ -246,7 +246,7 @@ T decode(InputStream inStream, Context context)
    * {@link org.apache.beam.sdk.runners.PipelineRunner}
    * implementations.
    */
-  boolean isRegisterByteSizeObserverCheap(T value);
+  public abstract boolean isRegisterByteSizeObserverCheap(T value);
 
   /**
    * Returns whether {@link #registerByteSizeObserver} cheap enough to
@@ -259,7 +259,7 @@ T decode(InputStream inStream, Context context)
    * implementations.
    */
   @Deprecated
-  boolean isRegisterByteSizeObserverCheap(T value, Context context);
+  public abstract boolean isRegisterByteSizeObserverCheap(T value, Context context);
 
   /**
    * Notifies the {@code ElementByteSizeObserver} about the byte size
@@ -269,7 +269,7 @@ T decode(InputStream inStream, Context context)
    * {@link org.apache.beam.sdk.runners.PipelineRunner}
    * implementations.
    */
-  void registerByteSizeObserver(
+  public abstract void registerByteSizeObserver(
       T value, ElementByteSizeObserver observer)
       throws Exception;
 
@@ -282,7 +282,7 @@ void registerByteSizeObserver(
    * implementations.
    */
   @Deprecated
-  void registerByteSizeObserver(
+  public abstract void registerByteSizeObserver(
       T value, ElementByteSizeObserver observer, Context context)
       throws Exception;
 
@@ -290,13 +290,13 @@ void registerByteSizeObserver(
    * Returns the {@link TypeDescriptor} for the type encoded.
    */
   @Experimental(Kind.CODER_TYPE_ENCODING)
-  TypeDescriptor<T> getEncodedTypeDescriptor();
+  public abstract TypeDescriptor<T> getEncodedTypeDescriptor();
 
   /**
    * Exception thrown by {@link Coder#verifyDeterministic()} if the encoding is
    * not deterministic, including details of why the encoding is not deterministic.
    */
-  class NonDeterministicException extends Exception {
+  public static class NonDeterministicException extends Exception {
     private Coder<?> coder;
     private List<String> reasons;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
index 0031698c0353..2a1d792b7603 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
@@ -17,6 +17,8 @@
  */
 package org.apache.beam.sdk.coders;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import com.google.common.base.MoreObjects;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -70,7 +72,12 @@ private CoderFactories() { } // Static utility class
    * will produce a {@code Coder<List<X>>} for any {@code Coder Coder<X>}.
    */
   public static <T> CoderFactory fromStaticMethods(Class<T> clazz) {
-    return new CoderFactoryFromStaticMethods(clazz);
+    checkArgument(
+        Coder.class.isAssignableFrom(clazz),
+        "%s is not a subtype of %s",
+        clazz.getName(),
+        Coder.class.getSimpleName());
+    return new CoderFactoryFromStaticMethods((Class<? extends Coder>) clazz);
   }
 
   /**
@@ -142,7 +149,7 @@ public List<Object> getInstanceComponents(Object value) {
      * Returns a CoderFactory that invokes the given static factory method
      * to create the Coder.
      */
-    private CoderFactoryFromStaticMethods(Class<?> coderClazz) {
+    private CoderFactoryFromStaticMethods(Class<? extends Coder> coderClazz) {
       this.factoryMethod = getFactoryMethod(coderClazz);
       this.getComponentsMethod = getInstanceComponentsMethod(coderClazz);
     }
@@ -203,8 +210,8 @@ private Method getFactoryMethod(Class<?> coderClazz) {
      * each corresponding to an argument of the {@code of}
      * method.
      */
-    private <T> Method getInstanceComponentsMethod(Class<?> coderClazz) {
-      TypeDescriptor<?> coderType = TypeDescriptor.of(coderClazz);
+    private <T, CoderT extends Coder> Method getInstanceComponentsMethod(Class<CoderT> coderClazz) {
+      TypeDescriptor<CoderT> coderType = TypeDescriptor.of(coderClazz);
       TypeDescriptor<T> argumentType = getCodedType(coderType);
 
       // getInstanceComponents may be implemented in a superclass,
@@ -235,19 +242,13 @@ private <T> Method getInstanceComponentsMethod(Class<?> coderClazz) {
      * If {@code coderType} is a subclass of {@link Coder} for a specific
      * type {@code T}, returns {@code T.class}. Otherwise, raises IllegalArgumentException.
      */
-    private <T> TypeDescriptor<T> getCodedType(TypeDescriptor<?> coderType) {
-      for (TypeDescriptor<?> ifaceType : coderType.getInterfaces()) {
-        if (ifaceType.getRawType().equals(Coder.class)) {
-          ParameterizedType coderIface = (ParameterizedType) ifaceType.getType();
-          @SuppressWarnings("unchecked")
-          TypeDescriptor<T> token =
-              (TypeDescriptor<T>) TypeDescriptor.of(coderIface.getActualTypeArguments()[0]);
-          return token;
-        }
-      }
-      throw new IllegalArgumentException(
-          "cannot build CoderFactory from class " + coderType
-          + ": does not implement Coder<T> for any T.");
+    private <T> TypeDescriptor<T> getCodedType(TypeDescriptor<? extends Coder> coderType) {
+      TypeDescriptor<?> coderSupertype = coderType.getSupertype(Coder.class);
+      ParameterizedType coderIface = (ParameterizedType) coderSupertype.getType();
+      @SuppressWarnings("unchecked")
+      TypeDescriptor<T> token =
+          (TypeDescriptor<T>) TypeDescriptor.of(coderIface.getActualTypeArguments()[0]);
+      return token;
     }
 
     @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
index cc39429a8e20..0c72618c2108 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
@@ -43,7 +43,7 @@
  *       expensive.</li>
  * </ul>
  */
-public abstract class StructuredCoder<T> implements Coder<T> {
+public abstract class StructuredCoder<T> extends Coder<T> {
   protected StructuredCoder() {}
 
   /**
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
index b937e84c5c96..07b3348fe101 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
@@ -25,8 +25,8 @@
 import com.google.common.base.Predicates;
 import com.google.common.collect.Iterables;
 import java.util.List;
-import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.BoundedPerElement;
 import org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement;
@@ -57,7 +57,7 @@ private static class SomeRestriction {}
   private abstract static class SomeRestrictionTracker
       implements RestrictionTracker<SomeRestriction> {}
 
-  private abstract static class SomeRestrictionCoder implements Coder<SomeRestriction> {}
+  private abstract static class SomeRestrictionCoder extends StructuredCoder<SomeRestriction> {}
 
   @Test
   public void testHasRestrictionTracker() throws Exception {

From 60f86db6ef211aedd7c7343842b68390ccf52d93 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 10:20:38 -0700
Subject: [PATCH 073/387] [BEAM-59] Fully delete IOChannelUtils

---
 .../apache/beam/sdk/util/IOChannelUtils.java  | 27 -------------------
 1 file changed, 27 deletions(-)
 delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java
deleted file mode 100644
index b658983eea87..000000000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IOChannelUtils.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import org.apache.beam.sdk.options.PipelineOptions;
-
-/** Do not use, being removed. */
-@Deprecated
-public class IOChannelUtils {
-  @Deprecated
-  public static void registerIOFactoriesAllowOverride(PipelineOptions options) {}
-}

From 9740c6ee2dca01c8de03383fbc62485ff6dd982d Mon Sep 17 00:00:00 2001
From: Daniel Halperin <dhalperi@users.noreply.github.com>
Date: Fri, 5 May 2017 12:10:16 -0700
Subject: [PATCH 074/387] [BEAM-2141] disable beam_PerformanceTests_Dataflow

Has not passed in Jenkins memory, at least multiple weeks.
---
 .test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
index 51c73f34d59e..2c8f7ef182bf 100644
--- a/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
@@ -40,4 +40,7 @@ job('beam_PerformanceTests_Dataflow'){
     ]
 
     common_job_properties.buildPerformanceTest(delegate, argMap)
+
+    // [BEAM-2141] Perf tests do not pass.
+    disabled()
 }

From c640e7437871313303636def48d63acd678a3430 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 5 May 2017 10:36:40 -0700
Subject: [PATCH 075/387] Move PipelineRunner to toplevel sdk package
 (automated refactor)

This allows excluding the runner-author-only sdk.runners package from the
public javadoc.
---
 .../java/org/apache/beam/runners/apex/ApexRunner.java    | 2 +-
 .../apache/beam/runners/apex/ApexRunnerRegistrar.java    | 2 +-
 .../org/apache/beam/runners/apex/TestApexRunner.java     | 2 +-
 .../main/java/org/apache/beam/runners/core/OldDoFn.java  | 4 ++--
 .../org/apache/beam/runners/direct/DirectRegistrar.java  | 2 +-
 .../org/apache/beam/runners/direct/DirectRunner.java     | 2 +-
 .../apache/beam/runners/direct/DoFnLifecycleManager.java | 2 +-
 .../org/apache/beam/runners/direct/DirectRunnerTest.java | 2 +-
 .../java/org/apache/beam/runners/flink/FlinkRunner.java  | 2 +-
 .../apache/beam/runners/flink/FlinkRunnerRegistrar.java  | 2 +-
 .../org/apache/beam/runners/flink/TestFlinkRunner.java   | 2 +-
 .../beam/runners/dataflow/DataflowPipelineRegistrar.java | 2 +-
 .../org/apache/beam/runners/dataflow/DataflowRunner.java | 2 +-
 .../runners/dataflow/testing/TestDataflowRunner.java     | 2 +-
 .../java/org/apache/beam/runners/spark/SparkRunner.java  | 2 +-
 .../apache/beam/runners/spark/SparkRunnerDebugger.java   | 2 +-
 .../apache/beam/runners/spark/SparkRunnerRegistrar.java  | 2 +-
 .../org/apache/beam/runners/spark/TestSparkRunner.java   | 2 +-
 .../core/src/main/java/org/apache/beam/sdk/Pipeline.java | 1 -
 .../apache/beam/sdk/{runners => }/PipelineRunner.java    | 4 +---
 .../src/main/java/org/apache/beam/sdk/coders/Coder.java  | 9 +++++----
 .../java/org/apache/beam/sdk/coders/package-info.java    | 2 +-
 .../src/main/java/org/apache/beam/sdk/io/AvroSource.java | 2 +-
 .../org/apache/beam/sdk/options/PipelineOptions.java     | 2 +-
 .../apache/beam/sdk/options/PipelineOptionsFactory.java  | 2 +-
 .../src/main/java/org/apache/beam/sdk/package-info.java  | 2 +-
 .../beam/sdk/runners/PTransformOverrideFactory.java      | 1 +
 .../apache/beam/sdk/runners/PipelineRunnerRegistrar.java | 1 +
 .../java/org/apache/beam/sdk/state/StateContext.java     | 3 ++-
 .../java/org/apache/beam/sdk/testing/CrashingRunner.java | 2 +-
 .../java/org/apache/beam/sdk/testing/NeedsRunner.java    | 2 +-
 .../main/java/org/apache/beam/sdk/testing/PAssert.java   | 2 +-
 .../java/org/apache/beam/sdk/testing/TestStream.java     | 2 +-
 .../apache/beam/sdk/transforms/CombineWithContext.java   | 3 ++-
 .../main/java/org/apache/beam/sdk/transforms/DoFn.java   | 6 +++---
 .../org/apache/beam/sdk/transforms/Materialization.java  | 2 +-
 .../main/java/org/apache/beam/sdk/transforms/ParDo.java  | 2 +-
 .../main/java/org/apache/beam/sdk/transforms/View.java   | 2 +-
 .../java/org/apache/beam/sdk/values/PCollectionView.java | 3 ++-
 .../src/test/java/org/apache/beam/sdk/PipelineTest.java  | 1 -
 .../beam/sdk/options/PipelineOptionsFactoryTest.java     | 2 +-
 .../org/apache/beam/sdk/runners/PipelineRunnerTest.java  | 1 +
 .../org/apache/beam/sdk/testing/CrashingRunnerTest.java  | 2 +-
 .../sdk/transforms/display/DisplayDataEvaluator.java     | 2 +-
 .../org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java  | 2 +-
 .../org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java  | 2 +-
 .../apache/beam/sdk/io/gcp/datastore/DatastoreV1.java    | 3 ++-
 .../java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 2 +-
 .../src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java  | 4 ++--
 49 files changed, 59 insertions(+), 55 deletions(-)
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/{runners => }/PipelineRunner.java (94%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index ede41bc4c351..a50e10ef53d8 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -43,6 +43,7 @@
 import org.apache.beam.runners.core.construction.PrimitiveCreate;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.ListCoder;
@@ -50,7 +51,6 @@
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
 import org.apache.beam.sdk.transforms.Create;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java
index 8cde692f8c15..2aa34ef15789 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunnerRegistrar.java
@@ -20,9 +20,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
index a64ac549ec70..e068db086442 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
@@ -19,9 +19,9 @@
 
 import java.io.IOException;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.joda.time.Duration;
 
 /**
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java
index 419c83779057..41bb598795e6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java
@@ -18,9 +18,9 @@
 package org.apache.beam.runners.core;
 
 import java.io.Serializable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFnTester;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -71,7 +71,7 @@ public abstract class Context {
 
     /**
      * Returns the {@code PipelineOptions} specified with the
-     * {@link org.apache.beam.sdk.runners.PipelineRunner}
+     * {@link PipelineRunner}
      * invoking this {@code OldDoFn}.  The {@code PipelineOptions} will
      * be the default running via {@link DoFnTester}.
      */
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
index 9a34d3d9dfb4..3e69e2b527d2 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
@@ -19,9 +19,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index 984598a6faaf..b0ce5eb02845 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -34,12 +34,12 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.metrics.MetricResults;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.runners.PTransformOverride;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java
index cd644a66585b..0e30e5c07bb7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java
@@ -26,7 +26,7 @@
 import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.Setup;
 import org.apache.beam.sdk.transforms.DoFn.Teardown;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index 83881fc46f24..428c6fc43d71 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -43,6 +43,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CustomCoder;
@@ -55,7 +56,6 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Count;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
index a5972efa1a15..80ef7bb32acd 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
@@ -31,10 +31,10 @@
 import java.util.TreeSet;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.View;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java
index 681459a819d7..ebc6adccf76c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java
@@ -20,9 +20,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/TestFlinkRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/TestFlinkRunner.java
index 8f50105a55b9..01b67e517d8e 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/TestFlinkRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/TestFlinkRunner.java
@@ -20,10 +20,10 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.util.UserCodeException;
 
 /**
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
index 5bd3bcd6eb9a..b6802bb4b0aa 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
@@ -20,9 +20,9 @@
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 4198baea7517..5278a4a576d0 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -77,6 +77,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
@@ -102,7 +103,6 @@
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.Combine;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
index c238d80bc933..ce9191586d1f 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
@@ -38,8 +38,8 @@
 import org.apache.beam.runners.dataflow.util.MonitoringUtil.JobMessagesHandler;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 16d9fdcfbbe7..8c02f0f0e90d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -41,13 +41,13 @@
 import org.apache.beam.runners.spark.translation.streaming.SparkRunnerStreamingContextFactory;
 import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.WatermarksListener;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
index 11c52c7a2ff3..6c74146bd61e 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerDebugger.java
@@ -25,9 +25,9 @@
 import org.apache.beam.runners.spark.translation.TransformTranslator;
 import org.apache.beam.runners.spark.translation.streaming.StreamingTransformTranslator;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.streaming.api.java.JavaStreamingContext;
 import org.joda.time.Duration;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
index bf926dc54ecc..e2e5ceba0221 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
@@ -20,9 +20,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 
 /**
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index ada7fb58fd7b..6d10b7588a42 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -39,13 +39,13 @@
 import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PBegin;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index 1769c0522c9c..f4da6ad9c1c2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -38,7 +38,6 @@
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.PTransformReplacement;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.Create;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
similarity index 94%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
index 229e04ffa12d..18e79eb7fa27 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunner.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
@@ -15,12 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.runners;
+package org.apache.beam.sdk;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index 169e448623c7..061e9e535338 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -29,6 +29,7 @@
 import java.util.Arrays;
 import java.util.List;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
@@ -243,7 +244,7 @@ public abstract T decode(InputStream inStream, Context context)
    * constant time (or lazily).
    *
    * <p>Not intended to be called by user code, but instead by
-   * {@link org.apache.beam.sdk.runners.PipelineRunner}
+   * {@link PipelineRunner}
    * implementations.
    */
   public abstract boolean isRegisterByteSizeObserverCheap(T value);
@@ -255,7 +256,7 @@ public abstract T decode(InputStream inStream, Context context)
    * constant time (or lazily).
    *
    * <p>Not intended to be called by user code, but instead by
-   * {@link org.apache.beam.sdk.runners.PipelineRunner}
+   * {@link PipelineRunner}
    * implementations.
    */
   @Deprecated
@@ -266,7 +267,7 @@ public abstract T decode(InputStream inStream, Context context)
    * of the encoded value using this {@code Coder}.
    *
    * <p>Not intended to be called by user code, but instead by
-   * {@link org.apache.beam.sdk.runners.PipelineRunner}
+   * {@link PipelineRunner}
    * implementations.
    */
   public abstract void registerByteSizeObserver(
@@ -278,7 +279,7 @@ public abstract void registerByteSizeObserver(
    * of the encoded value using this {@code Coder}.
    *
    * <p>Not intended to be called by user code, but instead by
-   * {@link org.apache.beam.sdk.runners.PipelineRunner}
+   * {@link PipelineRunner}
    * implementations.
    */
   @Deprecated
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java
index 84e59e018b45..5693077759c4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/package-info.java
@@ -27,7 +27,7 @@
  * machines.
  *
  * <p>Exactly when PCollection elements are encoded during execution depends on which
- * {@link org.apache.beam.sdk.runners.PipelineRunner} is being used and how that runner
+ * {@link org.apache.beam.sdk.PipelineRunner} is being used and how that runner
  * chooses to execute the pipeline. As such, Beam requires that all PCollections have an
  * appropriate Coder in case it becomes necessary. In many cases, the Coder can be inferred from
  * the available Java type
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
index 96d21c674658..61bc4a4f1563 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
@@ -48,12 +48,12 @@
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.reflect.ReflectData;
 import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
index 37081fde1a79..9a4d25ae3797 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
@@ -30,9 +30,9 @@
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.concurrent.ThreadSafe;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.ProxyInvocationHandler.Deserializer;
 import org.apache.beam.sdk.options.ProxyInvocationHandler.Serializer;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
 import org.joda.time.DateTimeUtils;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
index 0f6bedfc8b9d..c0990cb108a8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java
@@ -73,8 +73,8 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import javax.annotation.Nonnull;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.Validation.Required;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.util.StringUtils;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java
index 166c27651dc2..995bcb900854 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/package-info.java
@@ -27,7 +27,7 @@
  * the PTransforms consume and produce.
  *
  * <p>Each Pipeline has a
- * {@link org.apache.beam.sdk.runners.PipelineRunner} to specify
+ * {@link org.apache.beam.sdk.PipelineRunner} to specify
  * where and how it should run after pipeline construction is complete.
  *
  */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
index 5f012dedca06..58314c40058c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java
@@ -21,6 +21,7 @@
 
 import com.google.auto.value.AutoValue;
 import java.util.Map;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.annotations.Internal;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java
index 41fd6f0029ba..c6303dded4ec 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PipelineRunnerRegistrar.java
@@ -19,6 +19,7 @@
 
 import com.google.auto.service.AutoService;
 import java.util.ServiceLoader;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Internal;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
index 52177cc0a41c..b7dd8e2681cf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.state;
 
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -30,7 +31,7 @@
 public interface StateContext<W extends BoundedWindow> {
   /**
    * Returns the {@code PipelineOptions} specified with the
-   * {@link org.apache.beam.sdk.runners.PipelineRunner}.
+   * {@link PipelineRunner}.
    */
   PipelineOptions getPipelineOptions();
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CrashingRunner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CrashingRunner.java
index b3360ae6583e..0dea27460470 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CrashingRunner.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CrashingRunner.java
@@ -20,8 +20,8 @@
 
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 
 /**
  * A {@link PipelineRunner} that applies no overrides and throws an exception on calls to
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/NeedsRunner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/NeedsRunner.java
index 5f22bee631e8..8ebc76c66d58 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/NeedsRunner.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/NeedsRunner.java
@@ -18,7 +18,7 @@
 
 package org.apache.beam.sdk.testing;
 
-import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.PipelineRunner;
 
 /**
  * Category tag for validation tests which utilize {@link TestPipeline} for execution and expect
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
index b5d7db5fc204..6e2b8c66fa41 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java
@@ -34,6 +34,7 @@
 import java.util.NoSuchElementException;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.IterableCoder;
@@ -42,7 +43,6 @@
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
index 446c8a5d1974..d41b9ef5dc23 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
@@ -25,8 +25,8 @@
 import com.google.common.collect.ImmutableList;
 import java.util.List;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.PBegin;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineWithContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineWithContext.java
index 9ae19f8bfdd1..a7601b3cfc2c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineWithContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineWithContext.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.transforms;
 
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -36,7 +37,7 @@ public class CombineWithContext {
   public abstract static class Context {
     /**
      * Returns the {@code PipelineOptions} specified with the
-     * {@link org.apache.beam.sdk.runners.PipelineRunner}
+     * {@link PipelineRunner}
      * invoking this {@code KeyedCombineFn}.
      */
     public abstract PipelineOptions getPipelineOptions();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index befba1dd71d7..e711ac2f297a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -88,7 +88,7 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
   public abstract class StartBundleContext {
     /**
      * Returns the {@code PipelineOptions} specified with the {@link
-     * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+     * org.apache.beam.sdk.PipelineRunner} invoking this {@code DoFn}. The {@code
      * PipelineOptions} will be the default running via {@link DoFnTester}.
      */
     public abstract PipelineOptions getPipelineOptions();
@@ -100,7 +100,7 @@ public abstract class StartBundleContext {
   public abstract class FinishBundleContext {
     /**
      * Returns the {@code PipelineOptions} specified with the {@link
-     * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+     * org.apache.beam.sdk.PipelineRunner} invoking this {@code DoFn}. The {@code
      * PipelineOptions} will be the default running via {@link DoFnTester}.
      */
     public abstract PipelineOptions getPipelineOptions();
@@ -136,7 +136,7 @@ public abstract <T> void output(
   public abstract class WindowedContext {
     /**
      * Returns the {@code PipelineOptions} specified with the {@link
-     * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+     * org.apache.beam.sdk.PipelineRunner} invoking this {@code DoFn}. The {@code
      * PipelineOptions} will be the default running via {@link DoFnTester}.
      */
     public abstract PipelineOptions getPipelineOptions();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java
index 6fb8c297f547..0d02b32d3cc3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materialization.java
@@ -18,8 +18,8 @@
 
 package org.apache.beam.sdk.transforms;
 
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.runners.PipelineRunner;
 
 /**
  * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index c45311a7b0ab..e67dbe153a5e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -29,10 +29,10 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
 import org.apache.beam.sdk.transforms.display.DisplayData;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
index d7b81455b63f..bcbdb24d5290 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
@@ -19,10 +19,10 @@
 
 import java.util.List;
 import java.util.Map;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.sdk.values.KV;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
index f89041a6124e..7d874124a9bd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
@@ -19,6 +19,7 @@
 
 import java.io.Serializable;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.annotations.Internal;
@@ -36,7 +37,7 @@
  *
  * <p>A {@link PCollectionView} should always be the output of a
  * {@link org.apache.beam.sdk.transforms.PTransform}. It is the joint responsibility of
- * this transform and each {@link org.apache.beam.sdk.runners.PipelineRunner} to implement
+ * this transform and each {@link PipelineRunner} to implement
  * the view in a runner-specific manner.
  *
  * <p>The most common case is using the {@link View} transforms to prepare a {@link PCollection}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
index d6b527ccf6ee..2cc3f04203d1 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java
@@ -44,7 +44,6 @@
 import org.apache.beam.sdk.runners.PTransformMatcher;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.testing.CrashingRunner;
 import org.apache.beam.sdk.testing.ExpectedLogs;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
index 76a5f18c0397..d40b5fcc0dc6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java
@@ -53,7 +53,7 @@
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
 import org.apache.beam.sdk.testing.CrashingRunner;
 import org.apache.beam.sdk.testing.ExpectedLogs;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java
index 76d8627b19ed..c5d7fbf7cf23 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java
@@ -19,6 +19,7 @@
 
 import static org.junit.Assert.assertTrue;
 
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.CrashingRunner;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CrashingRunnerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CrashingRunnerTest.java
index c66aa50cb57f..62c5134ce67a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CrashingRunnerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CrashingRunnerTest.java
@@ -22,9 +22,9 @@
 
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.Create;
 import org.junit.Rule;
 import org.junit.Test;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
index 8f57f45b0ac4..6e8ca5e5db2f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
@@ -21,10 +21,10 @@
 import java.util.Objects;
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 6e410c7b6c28..304864a7ca85 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -43,6 +43,7 @@
 import java.util.Map;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
@@ -66,7 +67,6 @@
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 0e97c1254d93..846d07bb78a2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -42,6 +42,7 @@
 import java.util.NoSuchElementException;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
@@ -52,7 +53,6 @@
 import org.apache.beam.sdk.io.range.ByteKeyRange;
 import org.apache.beam.sdk.io.range.ByteKeyRangeTracker;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index fd4fccf0ba1d..d6464dd95855 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -69,6 +69,7 @@
 import java.util.List;
 import java.util.NoSuchElementException;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.SerializableCoder;
@@ -192,7 +193,7 @@
  * by providing the host port information through {@code withLocalhost("host:port"} for all the
  * above transforms. In such a case, all the Cloud Datastore API calls are directed to the Emulator.
  *
- * @see org.apache.beam.sdk.runners.PipelineRunner
+ * @see PipelineRunner
  */
 public class DatastoreV1 {
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index 51da111ccd24..ac6cb44be291 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -31,6 +31,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
@@ -44,7 +45,6 @@
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
index a34fb0f3cda0..7255a94357eb 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
@@ -26,6 +26,7 @@
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.ValidationEventHandler;
+import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.CompressedSource;
 import org.apache.beam.sdk.io.FileBasedSink;
@@ -34,7 +35,6 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
-import org.apache.beam.sdk.runners.PipelineRunner;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.values.PBegin;
@@ -102,7 +102,7 @@ public class XmlIO {
    *
    * <h3>Permissions</h3>
    *
-   * <p>Permission requirements depend on the {@link org.apache.beam.sdk.runners.PipelineRunner
+   * <p>Permission requirements depend on the {@link PipelineRunner
    * PipelineRunner} that is used to execute the Beam pipeline. Please refer to the documentation of
    * corresponding {@link PipelineRunner PipelineRunners} for more details.
    *

From b64f9acb116e92fb9ad35304a770aae92338a404 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 5 May 2017 10:33:11 -0700
Subject: [PATCH 076/387] Exclude sdk.runners from javadoc

---
 sdks/java/javadoc/ant.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sdks/java/javadoc/ant.xml b/sdks/java/javadoc/ant.xml
index 288f48bf9baa..9a1880acaa17 100644
--- a/sdks/java/javadoc/ant.xml
+++ b/sdks/java/javadoc/ant.xml
@@ -90,6 +90,7 @@
       <excludepackage name="org.apache.beam.runners.spark.examples.*"/>
       <excludepackage name="org.apache.beam.runners.spark.translation.*"/>
       <excludepackage name="org.apache.beam.sdk.transforms.reflect.*"/>
+      <excludepackage name="org.apache.beam.sdk.runners.*"/>
       <excludepackage name="org.apache.beam.sdk.util.*"/>
     </javadoc>
     <!-- make a jar file to push to central -->

From 15be5df5d83dd6d151e4b2ace0df8286a378b008 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 5 May 2017 10:39:26 -0700
Subject: [PATCH 077/387] Javadoc that the sdk.runners package is internal

---
 .../main/java/org/apache/beam/sdk/runners/package-info.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java
index bb6cce6079cb..272693643d60 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java
@@ -16,8 +16,8 @@
  * limitations under the License.
  */
 /**
- * Defines different data processing backends (aka, runners) for executing Beam Pipelines.
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
 
- * <p>The runner is specified as part of the {@link org.apache.beam.sdk.options.PipelineOptions}.
+ * <p>Internals for use by runners.
  */
 package org.apache.beam.sdk.runners;

From c6fe1f92fd0a49c58649ffb1dd40f53f14e098b9 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Fri, 5 May 2017 11:43:20 -0700
Subject: [PATCH 078/387] Use uuid for temp directory

---
 sdks/python/apache_beam/io/fileio.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 49562f796d59..bfed96ea59e2 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -23,6 +23,7 @@
 import os
 import re
 import time
+import uuid
 
 from apache_beam.internal import util
 from apache_beam.io import iobase
@@ -155,8 +156,7 @@ def _create_temp_dir(self, file_path_prefix):
                          'at least two components.',
                          file_path_prefix)
     path_components = [base_path,
-                       'beam-temp-' + last_component + time.strftime(
-                           '-%Y-%m-%d_%H-%M-%S')]
+                       'beam-temp-' + last_component + '-' + uuid.uuid1().hex]
     return FileSystems.join(*path_components)
 
   @check_accessible(['file_path_prefix', 'file_name_suffix'])

From 655947b597972d9fd6e1d3a777970b0b1152fa05 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 5 May 2017 10:08:32 -0700
Subject: [PATCH 079/387] Re-add AtomicCoder

This is a moderately useful base class for coders which take no
configuration.
---
 .../apache/beam/sdk/coders/AtomicCoder.java   | 86 +++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
new file mode 100644
index 000000000000..528cfb06d47b
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.coders;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A {@link Coder} that has no component {@link Coder Coders} or other state.
+ *
+ * <p>Note that, unless the behavior is overridden, atomic coders are presumed to be deterministic.
+ *
+ * <p>All atomic coders of the same class are considered to be equal to each other. As a result,
+ * an {@link AtomicCoder} should have no associated state.
+ *
+ * @param <T> the type of the values being transcoded
+ */
+public abstract class AtomicCoder<T> extends StructuredCoder<T> {
+  /**
+   * Returns an empty list.
+   *
+   * <p>{@link CoderFactories#fromStaticMethods(Class)} builds a {@link CoderFactory} from the
+   * {@code #of()} method and this method, used to determine the components of an object. Because
+   * {@link AtomicCoder} has no components, always returns an empty list.
+   *
+   * @param exampleValue unused, but part of the latent interface expected by {@link
+   *     CoderFactories#fromStaticMethods}
+   */
+  public static <T> List<Object> getInstanceComponents(T exampleValue) {
+    return Collections.emptyList();
+  }
+
+  /**
+   * {@inheritDoc}.
+   *
+   * @throws NonDeterministicException
+   */
+  @Override
+  public void verifyDeterministic() throws NonDeterministicException {}
+
+  @Override
+  public List<? extends Coder<?>> getCoderArguments() {
+    return null;
+  }
+
+  /**
+   * {@inheritDoc}.
+   *
+   * @return the empty {@link List}.
+   */
+  @Override
+  public final List<? extends Coder<?>> getComponents() {
+    return Collections.emptyList();
+  }
+
+  /**
+   * {@inheritDoc}.
+   *
+   * @return true if the other object has the same class as this {@link AtomicCoder}.
+   */
+  @Override
+  public final boolean equals(Object other) {
+    return other != null && this.getClass().equals(other.getClass());
+  }
+
+  @Override
+  public final int hashCode() {
+    return this.getClass().hashCode();
+  }
+}

From 63258c6986866b5bef58043b056d5c0dfec7303f Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 5 May 2017 10:10:40 -0700
Subject: [PATCH 080/387] Add default implementations of Coder methods to Coder

Remove from StructuredCoder. These are sensible defaults implemented in
terms of other Coder methods.
---
 .../org/apache/beam/sdk/coders/Coder.java     | 121 +++++++++++++++---
 .../beam/sdk/coders/StructuredCoder.java      |  67 +---------
 2 files changed, 108 insertions(+), 80 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index 061e9e535338..41e83acf6e50 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -22,6 +22,9 @@
 import com.google.common.base.Joiner;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.CountingOutputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -205,6 +208,30 @@ public abstract T decode(InputStream inStream, Context context)
    */
   public abstract void verifyDeterministic() throws Coder.NonDeterministicException;
 
+  /**
+   * Verifies all of the provided coders are deterministic. If any are not, throws a {@link
+   * NonDeterministicException} for the {@code target} {@link Coder}.
+   */
+  public static void verifyDeterministic(Coder<?> target, String message, Iterable<Coder<?>> coders)
+      throws NonDeterministicException {
+    for (Coder<?> coder : coders) {
+      try {
+        coder.verifyDeterministic();
+      } catch (NonDeterministicException e) {
+        throw new NonDeterministicException(target, message, e);
+      }
+    }
+  }
+
+  /**
+   * Verifies all of the provided coders are deterministic. If any are not, throws a {@link
+   * NonDeterministicException} for the {@code target} {@link Coder}.
+   */
+  public static void verifyDeterministic(Coder<?> target, String message, Coder<?>... coders)
+      throws NonDeterministicException {
+    verifyDeterministic(target, message, Arrays.asList(coders));
+  }
+
   /**
    * Returns {@code true} if this {@link Coder} is injective with respect to {@link Objects#equals}.
    *
@@ -214,28 +241,50 @@ public abstract T decode(InputStream inStream, Context context)
    * <p>This condition is most notably false for arrays. More generally, this condition is false
    * whenever {@code equals()} compares object identity, rather than performing a
    * semantic/structural comparison.
+   *
+   * <p>By default, returns false.
    */
-  public abstract boolean consistentWithEquals();
+  public boolean consistentWithEquals() {
+    return false;
+  }
 
   /**
-   * Returns an object with an {@code Object.equals()} method that represents structural equality
-   * on the argument.
+   * Returns an object with an {@code Object.equals()} method that represents structural equality on
+   * the argument.
    *
    * <p>For any two values {@code x} and {@code y} of type {@code T}, if their encoded bytes are the
    * same, then it must be the case that {@code structuralValue(x).equals(@code structuralValue(y)}.
    *
    * <p>Most notably:
+   *
    * <ul>
    *   <li>The structural value for an array coder should perform a structural comparison of the
-   *   contents of the arrays, rather than the default behavior of comparing according to object
-   *   identity.
-   *   <li>The structural value for a coder accepting {@code null} should be a proper object with
-   *   an {@code equals()} method, even if the input value is {@code null}.
+   *       contents of the arrays, rather than the default behavior of comparing according to object
+   *       identity.
+   *   <li>The structural value for a coder accepting {@code null} should be a proper object with an
+   *       {@code equals()} method, even if the input value is {@code null}.
    * </ul>
    *
    * <p>See also {@link #consistentWithEquals()}.
+   *
+   * <p>By default, if this coder is {@link #consistentWithEquals()}, and the value is not null,
+   * returns the provided object. Otherwise, encodes the value into a {@code byte[]}, and returns
+   * an object that performs array equality on the encoded bytes.
    */
-  public abstract Object structuralValue(T value);
+  public Object structuralValue(T value) {
+    if (value != null && consistentWithEquals()) {
+      return value;
+    } else {
+      try {
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
+        encode(value, os, Context.OUTER);
+        return new StructuralByteArray(os.toByteArray());
+      } catch (Exception exn) {
+        throw new IllegalArgumentException(
+            "Unable to encode element '" + value + "' with coder '" + this + "'.", exn);
+      }
+    }
+  }
 
   /**
    * Returns whether {@link #registerByteSizeObserver} cheap enough to
@@ -246,21 +295,44 @@ public abstract T decode(InputStream inStream, Context context)
    * <p>Not intended to be called by user code, but instead by
    * {@link PipelineRunner}
    * implementations.
+   *
+   * <p>By default, returns false. The default {@link #registerByteSizeObserver} implementation
+   *         invokes {@link #getEncodedElementByteSize} which requires re-encoding an element
+   *         unless it is overridden. This is considered expensive.
    */
-  public abstract boolean isRegisterByteSizeObserverCheap(T value);
+  public boolean isRegisterByteSizeObserverCheap(T value) {
+    return isRegisterByteSizeObserverCheap(value, Context.NESTED);
+  }
 
   /**
-   * Returns whether {@link #registerByteSizeObserver} cheap enough to
-   * call for every element, that is, if this {@code Coder} can
-   * calculate the byte size of the element to be coded in roughly
-   * constant time (or lazily).
+   * {@inheritDoc}
    *
    * <p>Not intended to be called by user code, but instead by
    * {@link PipelineRunner}
    * implementations.
+   *
+   * @return {@code false} unless it is overridden. {@link StructuredCoder#registerByteSizeObserver}
+   *         invokes {@link #getEncodedElementByteSize} which requires re-encoding an element
+   *         unless it is overridden. This is considered expensive.
    */
   @Deprecated
-  public abstract boolean isRegisterByteSizeObserverCheap(T value, Context context);
+  public boolean isRegisterByteSizeObserverCheap(T value, Context context) {
+    return false;
+  }
+
+  /**
+   * Returns the size in bytes of the encoded value using this coder.
+   */
+  protected long getEncodedElementByteSize(T value, Context context)
+      throws Exception {
+    try (CountingOutputStream os = new CountingOutputStream(ByteStreams.nullOutputStream())) {
+      encode(value, os, context);
+      return os.getCount();
+    } catch (Exception exn) {
+      throw new IllegalArgumentException(
+          "Unable to encode element '" + value + "' with coder '" + this + "'.", exn);
+    }
+  }
 
   /**
    * Notifies the {@code ElementByteSizeObserver} about the byte size
@@ -269,10 +341,14 @@ public abstract T decode(InputStream inStream, Context context)
    * <p>Not intended to be called by user code, but instead by
    * {@link PipelineRunner}
    * implementations.
+   *
+   * <p>By default, this notifies {@code observer} about the byte size
+   * of the encoded value using this coder as returned by {@link #getEncodedElementByteSize}.
    */
-  public abstract void registerByteSizeObserver(
-      T value, ElementByteSizeObserver observer)
-      throws Exception;
+  public void registerByteSizeObserver(T value, ElementByteSizeObserver observer)
+      throws Exception {
+    registerByteSizeObserver(value, observer, Context.NESTED);
+  }
 
   /**
    * Notifies the {@code ElementByteSizeObserver} about the byte size
@@ -283,15 +359,20 @@ public abstract void registerByteSizeObserver(
    * implementations.
    */
   @Deprecated
-  public abstract void registerByteSizeObserver(
+  public void registerByteSizeObserver(
       T value, ElementByteSizeObserver observer, Context context)
-      throws Exception;
+      throws Exception {
+    observer.update(getEncodedElementByteSize(value, context));
+  }
 
   /**
    * Returns the {@link TypeDescriptor} for the type encoded.
    */
   @Experimental(Kind.CODER_TYPE_ENCODING)
-  public abstract TypeDescriptor<T> getEncodedTypeDescriptor();
+  public TypeDescriptor<T> getEncodedTypeDescriptor(){
+    return (TypeDescriptor<T>)
+        TypeDescriptor.of(getClass()).resolveType(new TypeDescriptor<T>() {}.getType());
+  }
 
   /**
    * Exception thrown by {@link Coder#verifyDeterministic()} if the encoding is
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
index 0c72618c2108..437f10d0b1c0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
@@ -17,8 +17,6 @@
  */
 package org.apache.beam.sdk.coders;
 
-import com.google.common.io.ByteStreams;
-import com.google.common.io.CountingOutputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -26,13 +24,15 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
  * An abstract base class to implement a {@link Coder} that defines equality, hashing, and printing
  * via the class name and recursively using {@link #getComponents}.
  *
+ * <p>A {@link StructuredCoder} should be defined purely in terms of its component coders, and
+ * contain no additional configuration.
+ *
  * <p>To extend {@link StructuredCoder}, override the following methods as appropriate:
  *
  * <ul>
@@ -101,12 +101,14 @@ public String toString() {
     return builder.toString();
   }
 
+  @Override
   public void encode(T value, OutputStream outStream)
       throws CoderException, IOException {
     encode(value, outStream, Coder.Context.NESTED);
   }
 
   @Deprecated
+  @Override
   public void encodeOuter(T value, OutputStream outStream)
       throws CoderException, IOException {
     encode(value, outStream, Coder.Context.OUTER);
@@ -122,11 +124,13 @@ public void encode(T value, OutputStream outStream, Coder.Context context)
     }
   }
 
+  @Override
   public T decode(InputStream inStream) throws CoderException, IOException {
     return decode(inStream, Coder.Context.NESTED);
   }
 
   @Deprecated
+  @Override
   public T decodeOuter(InputStream inStream) throws CoderException, IOException {
     return decode(inStream, Coder.Context.OUTER);
   }
@@ -141,63 +145,6 @@ public T decode(InputStream inStream, Coder.Context context)
     }
   }
 
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code false} unless it is overridden. {@link StructuredCoder#registerByteSizeObserver}
-   *         invokes {@link #getEncodedElementByteSize} which requires re-encoding an element
-   *         unless it is overridden. This is considered expensive.
-   */
-  @Override
-  public boolean isRegisterByteSizeObserverCheap(T value) {
-    return isRegisterByteSizeObserverCheap(value, Context.NESTED);
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * @return {@code false} unless it is overridden. {@link StructuredCoder#registerByteSizeObserver}
-   *         invokes {@link #getEncodedElementByteSize} which requires re-encoding an element
-   *         unless it is overridden. This is considered expensive.
-   */
-  @Override
-  public boolean isRegisterByteSizeObserverCheap(T value, Context context) {
-    return false;
-  }
-
-  /**
-   * Returns the size in bytes of the encoded value using this coder.
-   */
-  protected long getEncodedElementByteSize(T value, Context context)
-      throws Exception {
-    try (CountingOutputStream os = new CountingOutputStream(ByteStreams.nullOutputStream())) {
-      encode(value, os, context);
-      return os.getCount();
-    } catch (Exception exn) {
-      throw new IllegalArgumentException(
-          "Unable to encode element '" + value + "' with coder '" + this + "'.", exn);
-    }
-  }
-
-  @Override
-  public void registerByteSizeObserver(T value, ElementByteSizeObserver observer)
-      throws Exception {
-    registerByteSizeObserver(value, observer, Context.NESTED);
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * <p>For {@link StructuredCoder} subclasses, this notifies {@code observer} about the byte size
-   * of the encoded value using this coder as returned by {@link #getEncodedElementByteSize}.
-   */
-  @Override
-  public void registerByteSizeObserver(
-      T value, ElementByteSizeObserver observer, Context context)
-      throws Exception {
-    observer.update(getEncodedElementByteSize(value, context));
-  }
-
   protected void verifyDeterministic(String message, Iterable<Coder<?>> coders)
       throws NonDeterministicException {
     for (Coder<?> coder : coders) {

From 987c2cbca433f3e0ff12a637f2b0474e772c6beb Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 5 May 2017 10:13:05 -0700
Subject: [PATCH 081/387] Reparent many Coders to Atomic or StructuredCoder

These coders do not take configuration, or take configuration only in
terms of other Coders, and are appropriate to reparent.
---
 .../translation/utils/ApexStreamTuple.java    |  5 ++-
 .../UnboundedReadFromBoundedSource.java       |  4 +-
 .../runners/core/construction/CodersTest.java | 14 +------
 .../core/construction/PCollectionsTest.java   |  3 +-
 .../core/ElementAndRestrictionCoder.java      | 17 +++++++-
 .../beam/runners/core/KeyedWorkItemCoder.java |  4 +-
 .../beam/runners/core/TimerInternals.java     |  6 +--
 .../direct/CloningBundleFactoryTest.java      | 10 ++---
 .../beam/runners/direct/DirectRunnerTest.java |  5 +--
 .../UnboundedReadEvaluatorFactoryTest.java    |  4 +-
 .../SingletonKeyedWorkItemCoder.java          |  4 +-
 .../runners/dataflow/BatchViewOverrides.java  |  8 ++--
 .../runners/dataflow/internal/IsmFormat.java  | 40 +++++++++++++++----
 .../dataflow/util/RandomAccessData.java       |  4 +-
 .../org/apache/beam/sdk/coders/AvroCoder.java | 19 +++++++++
 .../beam/sdk/coders/BigDecimalCoder.java      |  2 +-
 .../sdk/coders/BigEndianIntegerCoder.java     |  2 +-
 .../beam/sdk/coders/BigEndianLongCoder.java   |  4 +-
 .../beam/sdk/coders/BigIntegerCoder.java      |  4 +-
 .../apache/beam/sdk/coders/BitSetCoder.java   |  4 +-
 .../beam/sdk/coders/ByteArrayCoder.java       |  6 +--
 .../org/apache/beam/sdk/coders/ByteCoder.java |  2 +-
 .../beam/sdk/coders/CoderFactories.java       |  9 +++--
 .../apache/beam/sdk/coders/DoubleCoder.java   |  2 +-
 .../apache/beam/sdk/coders/DurationCoder.java |  2 +-
 .../apache/beam/sdk/coders/InstantCoder.java  |  2 +-
 .../org/apache/beam/sdk/coders/KvCoder.java   |  4 +-
 .../org/apache/beam/sdk/coders/ListCoder.java |  3 +-
 .../org/apache/beam/sdk/coders/MapCoder.java  |  2 +-
 .../apache/beam/sdk/coders/NullableCoder.java |  6 +--
 .../beam/sdk/coders/StringUtf8Coder.java      |  2 +-
 .../beam/sdk/coders/TextualIntegerCoder.java  |  2 +-
 .../apache/beam/sdk/coders/VarIntCoder.java   |  2 +-
 .../org/apache/beam/sdk/coders/VoidCoder.java |  4 +-
 .../org/apache/beam/sdk/io/FileBasedSink.java |  4 +-
 .../sdk/transforms/ApproximateQuantiles.java  | 26 +++++++++---
 .../apache/beam/sdk/transforms/Combine.java   | 12 ++++--
 .../beam/sdk/transforms/CombineFns.java       |  4 +-
 .../org/apache/beam/sdk/transforms/Count.java |  4 +-
 .../org/apache/beam/sdk/transforms/Mean.java  |  7 ++--
 .../org/apache/beam/sdk/transforms/Top.java   | 23 ++++++++++-
 .../beam/sdk/transforms/join/CoGbkResult.java |  2 +-
 .../beam/sdk/transforms/join/UnionCoder.java  |  7 ++--
 .../sdk/transforms/windowing/PaneInfo.java    | 10 ++++-
 .../org/apache/beam/sdk/util/BitSetCoder.java |  9 +++--
 .../apache/beam/sdk/util/WindowedValue.java   |  7 ++--
 .../beam/sdk/values/TimestampedValue.java     | 13 ++++--
 .../beam/sdk/values/ValueInSingleWindow.java  |  4 +-
 .../beam/sdk/values/ValueWithRecordId.java    |  4 +-
 .../beam/sdk/coders/CoderRegistryTest.java    |  4 +-
 .../beam/sdk/coders/DelegateCoderTest.java    | 25 ------------
 .../beam/sdk/coders/NullableCoderTest.java    |  2 +-
 .../beam/sdk/testing/CoderPropertiesTest.java | 37 +++++++++++++++--
 .../apache/beam/sdk/testing/PAssertTest.java  |  4 +-
 .../sdk/testing/SerializableMatchersTest.java |  4 +-
 .../beam/sdk/testing/WindowSupplierTest.java  |  6 +--
 .../beam/sdk/transforms/CombineFnsTest.java   |  4 +-
 .../beam/sdk/transforms/CombineTest.java      |  6 +--
 .../beam/sdk/transforms/CreateTest.java       |  6 +--
 .../beam/sdk/transforms/GroupByKeyTest.java   |  4 +-
 .../apache/beam/sdk/transforms/ParDoTest.java |  6 +--
 .../apache/beam/sdk/transforms/ViewTest.java  |  4 +-
 .../transforms/reflect/DoFnInvokersTest.java  |  6 +--
 .../apache/beam/sdk/util/CoderUtilsTest.java  |  4 +-
 .../beam/sdk/util/SerializableUtilsTest.java  |  4 +-
 .../extensions/protobuf/ByteStringCoder.java  |  4 +-
 .../sdk/io/gcp/bigquery/ShardedKeyCoder.java  | 10 ++---
 .../gcp/bigquery/TableDestinationCoder.java   |  4 +-
 .../io/gcp/bigquery/TableRowInfoCoder.java    |  4 +-
 .../io/gcp/bigquery/TableRowJsonCoder.java    |  4 +-
 .../io/gcp/bigquery/WriteBundlesToFiles.java  | 11 ++++-
 .../io/gcp/pubsub/PubsubUnboundedSink.java    |  4 +-
 .../io/gcp/pubsub/PubsubUnboundedSource.java  | 12 ++++--
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java   |  4 +-
 .../beam/sdk/io/hadoop/WritableCoder.java     | 18 +++++++++
 .../beam/sdk/io/hbase/HBaseMutationCoder.java |  4 +-
 .../beam/sdk/io/hbase/HBaseResultCoder.java   |  4 +-
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java |  6 +--
 .../beam/sdk/io/kafka/KafkaRecordCoder.java   |  4 +-
 .../sdk/io/kinesis/KinesisRecordCoder.java    |  4 +-
 .../org/apache/beam/sdk/io/xml/JAXBCoder.java | 18 +++++++++
 .../apache/beam/sdk/io/xml/JAXBCoderTest.java |  4 +-
 82 files changed, 369 insertions(+), 223 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java
index 4ce351b882f3..4aa6ee82017e 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java
@@ -31,7 +31,7 @@
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 
 /**
  * The common interface for all objects transmitted through streams.
@@ -149,7 +149,7 @@ public String toString() {
   /**
    * Coder for {@link ApexStreamTuple}.
    */
-  class ApexStreamTupleCoder<T> extends CustomCoder<ApexStreamTuple<T>> {
+  class ApexStreamTupleCoder<T> extends StructuredCoder<ApexStreamTuple<T>> {
     private static final long serialVersionUID = 1L;
     final Coder<T> valueCoder;
 
@@ -194,6 +194,7 @@ public List<? extends Coder<?>> getCoderArguments() {
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
       verifyDeterministic(
+          this,
           this.getClass().getSimpleName() + " requires a deterministic valueCoder",
           valueCoder);
     }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
index 0ea13b8235d4..1424b8be2518 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
@@ -35,10 +35,10 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.BoundedSource.BoundedReader;
 import org.apache.beam.sdk.io.Read;
@@ -203,7 +203,7 @@ public void finalizeCheckpoint() {}
     }
 
     @VisibleForTesting
-    static class CheckpointCoder<T> extends CustomCoder<Checkpoint<T>> {
+    static class CheckpointCoder<T> extends StructuredCoder<Checkpoint<T>> {
 
       // The coder for a list of residual elements and their timestamps
       private final Coder<List<TimestampedValue<T>>> elemsCoder;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
index 32a78faec70e..765723c9bb3d 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
@@ -30,11 +30,11 @@
 import java.io.Serializable;
 import java.util.HashSet;
 import java.util.Set;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.LengthPrefixCoder;
@@ -149,7 +149,7 @@ public void toAndFromProto() throws Exception {
 
     static class Record implements Serializable {}
 
-    private static class RecordCoder extends CustomCoder<Record> {
+    private static class RecordCoder extends AtomicCoder<Record> {
       @Override
       public void encode(Record value, OutputStream outStream, Context context)
           throws CoderException, IOException {}
@@ -159,16 +159,6 @@ public Record decode(InputStream inStream, Context context)
           throws CoderException, IOException {
         return new Record();
       }
-
-      @Override
-      public boolean equals(Object other) {
-        return other != null && getClass().equals(other.getClass());
-      }
-
-      @Override
-      public int hashCode() {
-        return getClass().hashCode();
-      }
     }
   }
 }
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
index c177c58cb6d6..2c45cbdb94b3 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
@@ -29,6 +29,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CustomCoder;
@@ -158,7 +159,7 @@ public boolean isCompatible(WindowFn<?, ?> other) {
 
     @Override
     public Coder<BoundedWindow> windowCoder() {
-      return new CustomCoder<BoundedWindow>() {
+      return new AtomicCoder<BoundedWindow>() {
         @Override public void verifyDeterministic() {}
 
         @Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
index 64c1e14156c0..83c4e6281d71 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
@@ -17,18 +17,20 @@
  */
 package org.apache.beam.runners.core;
 
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.List;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 
 /** A {@link Coder} for {@link ElementAndRestriction}. */
 @Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
 public class ElementAndRestrictionCoder<ElementT, RestrictionT>
-    extends CustomCoder<ElementAndRestriction<ElementT, RestrictionT>> {
+    extends StructuredCoder<ElementAndRestriction<ElementT, RestrictionT>> {
   private final Coder<ElementT> elementCoder;
   private final Coder<RestrictionT> restrictionCoder;
 
@@ -65,6 +67,17 @@ public ElementAndRestriction<ElementT, RestrictionT> decode(InputStream inStream
     return ElementAndRestriction.of(key, value);
   }
 
+  @Override
+  public List<? extends Coder<?>> getCoderArguments() {
+    return ImmutableList.of(elementCoder, restrictionCoder);
+  }
+
+  @Override
+  public void verifyDeterministic() throws NonDeterministicException {
+    elementCoder.verifyDeterministic();
+    restrictionCoder.verifyDeterministic();
+  }
+
   public Coder<ElementT> getElementCoder() {
     return elementCoder;
   }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
index fddf7fa478e1..e1872b542260 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
@@ -26,8 +26,8 @@
 import org.apache.beam.runners.core.TimerInternals.TimerDataCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
@@ -35,7 +35,7 @@
 /**
  * A {@link Coder} for {@link KeyedWorkItem KeyedWorkItems}.
  */
-public class KeyedWorkItemCoder<K, ElemT> extends CustomCoder<KeyedWorkItem<K, ElemT>> {
+public class KeyedWorkItemCoder<K, ElemT> extends StructuredCoder<KeyedWorkItem<K, ElemT>> {
   /**
    * Create a new {@link KeyedWorkItemCoder} with the provided key coder, element coder, and window
    * coder.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
index 21fe4301ea57..888c11f97313 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
@@ -27,9 +27,9 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -224,7 +224,7 @@ public int compareTo(TimerData that) {
   /**
    * A {@link Coder} for {@link TimerData}.
    */
-  class TimerDataCoder extends CustomCoder<TimerData> {
+  class TimerDataCoder extends StructuredCoder<TimerData> {
     private static final StringUtf8Coder STRING_CODER = StringUtf8Coder.of();
     private static final InstantCoder INSTANT_CODER = InstantCoder.of();
     private final Coder<? extends BoundedWindow> windowCoder;
@@ -266,7 +266,7 @@ public List<? extends Coder<?>> getCoderArguments() {
 
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
-      verifyDeterministic("window coder must be deterministic", windowCoder);
+      verifyDeterministic(this, "window coder must be deterministic", windowCoder);
     }
   }
 }
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java
index 7d037d124fd7..33d171e50449 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java
@@ -31,8 +31,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
@@ -173,7 +173,7 @@ public void keyedBundleDecodeFailsAddFails() {
   }
 
   static class Record {}
-  static class RecordNoEncodeCoder extends CustomCoder<Record> {
+  static class RecordNoEncodeCoder extends AtomicCoder<Record> {
 
     @Override
     public void encode(
@@ -192,7 +192,7 @@ public Record decode(
     }
   }
 
-  static class RecordNoDecodeCoder extends CustomCoder<Record> {
+  static class RecordNoDecodeCoder extends AtomicCoder<Record> {
     @Override
     public void encode(
         Record value,
@@ -208,7 +208,7 @@ public Record decode(
     }
   }
 
-  private static class RecordStructuralValueCoder extends CustomCoder<Record> {
+  private static class RecordStructuralValueCoder extends AtomicCoder<Record> {
     @Override
     public void encode(
         Record value,
@@ -240,7 +240,7 @@ public Object structuralValue(Record value) {
   }
 
   private static class RecordNotConsistentWithEqualsStructuralValueCoder
-      extends CustomCoder<Record> {
+      extends AtomicCoder<Record> {
     @Override
     public void encode(
         Record value,
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index 428c6fc43d71..0fe958515465 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -44,9 +44,9 @@
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
@@ -523,8 +523,7 @@ public void testUnencodableOutputFromUnboundedRead() {
     p.run();
   }
 
-  private static class LongNoDecodeCoder extends CustomCoder<Long> {
-
+  private static class LongNoDecodeCoder extends AtomicCoder<Long> {
     @Override
     public void encode(
         Long value, OutputStream outStream, Context context) throws IOException {
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index ceb078be3058..b9ba7f49df79 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -46,10 +46,10 @@
 import javax.annotation.Nullable;
 import org.apache.beam.runners.direct.UnboundedReadDeduplicator.NeverDeduplicator;
 import org.apache.beam.runners.direct.UnboundedReadEvaluatorFactory.UnboundedSourceShard;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.Read;
@@ -586,7 +586,7 @@ boolean isFinalized() {
       return finalized;
     }
 
-    public static class Coder extends CustomCoder<TestCheckpointMark> {
+    public static class Coder extends AtomicCoder<TestCheckpointMark> {
       @Override
       public void encode(
           TestCheckpointMark value,
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
index c73700f9dc23..f21869341a01 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
@@ -26,7 +26,7 @@
 import org.apache.beam.runners.core.KeyedWorkItemCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 
@@ -34,7 +34,7 @@
  * Singleton keyed work item coder.
  */
 public class SingletonKeyedWorkItemCoder<K, ElemT>
-    extends CustomCoder<SingletonKeyedWorkItem<K, ElemT>> {
+    extends StructuredCoder<SingletonKeyedWorkItem<K, ElemT>> {
   /**
    * Create a new {@link KeyedWorkItemCoder} with the provided key coder, element coder, and window
    * coder.
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index ef2bfed39e7c..ecd0365ed88c 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -50,11 +50,11 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.MapCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -1335,7 +1335,7 @@ protected Map<K, V2> delegate() {
    * A {@link Coder} for {@link TransformedMap}s.
    */
   static class TransformedMapCoder<K, V1, V2>
-      extends CustomCoder<TransformedMap<K, V1, V2>> {
+      extends StructuredCoder<TransformedMap<K, V1, V2>> {
     private final Coder<Function<V1, V2>> transformCoder;
     private final Coder<Map<K, V1>> originalMapCoder;
 
@@ -1373,8 +1373,8 @@ public List<? extends Coder<?>> getCoderArguments() {
     @Override
     public void verifyDeterministic()
         throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
-      verifyDeterministic("Expected transform coder to be deterministic.", transformCoder);
-      verifyDeterministic("Expected map coder to be deterministic.", originalMapCoder);
+      verifyDeterministic(this, "Expected transform coder to be deterministic.", transformCoder);
+      verifyDeterministic(this, "Expected map coder to be deterministic.", originalMapCoder);
     }
   }
 
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
index fbfe49aaa5ad..aed514a08c7c 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
@@ -32,14 +32,17 @@
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.dataflow.util.RandomAccessData;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.util.VarInt;
@@ -356,8 +359,9 @@ public List<Coder<?>> getCoderArguments() {
 
     @Override
     public void verifyDeterministic() throws Coder.NonDeterministicException {
-      verifyDeterministic("Key component coders expected to be deterministic.", keyComponentCoders);
-      verifyDeterministic("Value coder expected to be deterministic.", valueCoder);
+      verifyDeterministic(
+          this, "Key component coders expected to be deterministic.", keyComponentCoders);
+      verifyDeterministic(this, "Value coder expected to be deterministic.", valueCoder);
     }
 
     @Override
@@ -393,6 +397,28 @@ public Object structuralValue(IsmRecord<V> record) {
       }
       return super.structuralValue(record);
     }
+
+    @Override
+    public boolean equals(Object other) {
+      if (other == this) {
+        return true;
+      }
+      if (!(other instanceof IsmRecordCoder)) {
+        return false;
+      }
+      IsmRecordCoder<?> that = (IsmRecordCoder<?>) other;
+      return Objects.equals(this.numberOfShardKeyCoders, that.numberOfShardKeyCoders)
+          && Objects.equals(
+              this.numberOfMetadataShardKeyCoders, that.numberOfMetadataShardKeyCoders)
+          && Objects.equals(this.keyComponentCoders, that.keyComponentCoders)
+          && Objects.equals(this.valueCoder, that.valueCoder);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(
+          numberOfShardKeyCoders, numberOfMetadataShardKeyCoders, keyComponentCoders, valueCoder);
+    }
   }
 
   /**
@@ -450,7 +476,7 @@ public static Object getMetadataKey() {
    * A coder for metadata key component. Can be used to wrap key component coder allowing for
    * the metadata key component to be used as a place holder instead of an actual key.
    */
-  public static class MetadataKeyCoder<K> extends CustomCoder<K> {
+  public static class MetadataKeyCoder<K> extends StructuredCoder<K> {
     public static <K> MetadataKeyCoder<K> of(Coder<K> keyCoder) {
       checkNotNull(keyCoder);
       return new MetadataKeyCoder<>(keyCoder);
@@ -497,7 +523,7 @@ public List<Coder<?>> getCoderArguments() {
 
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
-      verifyDeterministic("Expected key coder to be deterministic", keyCoder);
+      verifyDeterministic(this, "Expected key coder to be deterministic", keyCoder);
     }
   }
 
@@ -584,7 +610,7 @@ public IsmShard withIndexOffset(long indexOffset) {
    *   <li>indexOffset (variable length long encoding)</li>
    * </ul>
    */
-  public static class IsmShardCoder extends CustomCoder<IsmShard> {
+  public static class IsmShardCoder extends AtomicCoder<IsmShard> {
     private static final IsmShardCoder INSTANCE = new IsmShardCoder();
 
     /** Returns an IsmShardCoder. */
@@ -649,7 +675,7 @@ public static KeyPrefix of(int sharedKeySize, int unsharedKeySize) {
   }
 
   /** A {@link Coder} for {@link KeyPrefix}. */
-  public static final class KeyPrefixCoder extends CustomCoder<KeyPrefix> {
+  public static final class KeyPrefixCoder extends AtomicCoder<KeyPrefix> {
     private static final KeyPrefixCoder INSTANCE = new KeyPrefixCoder();
 
     public static KeyPrefixCoder of() {
@@ -721,7 +747,7 @@ public static Footer of(long indexPosition, long bloomFilterPosition, long numbe
   }
 
   /** A {@link Coder} for {@link Footer}. */
-  public static final class FooterCoder extends CustomCoder<Footer> {
+  public static final class FooterCoder extends AtomicCoder<Footer> {
     private static final FooterCoder INSTANCE = new FooterCoder();
 
     public static FooterCoder of() {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
index 66548e25b55a..4e9451511d1a 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
@@ -30,10 +30,10 @@
 import java.util.Arrays;
 import java.util.Comparator;
 import javax.annotation.concurrent.NotThreadSafe;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.util.VarInt;
 
 /**
@@ -55,7 +55,7 @@ public class RandomAccessData {
    *
    * <p>This coder does not support encoding positive infinity.
    */
-  public static class RandomAccessDataCoder extends CustomCoder<RandomAccessData> {
+  public static class RandomAccessDataCoder extends AtomicCoder<RandomAccessData> {
     private static final RandomAccessDataCoder INSTANCE = new RandomAccessDataCoder();
 
     public static RandomAccessDataCoder of() {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
index 1e01f1a0bfe3..2aa2b44c63d7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
@@ -29,6 +29,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.SortedSet;
@@ -681,4 +682,22 @@ private static Field getField(Class<?> originalClazz, String name) {
       throw new IllegalArgumentException("Unable to get field " + name + " from " + originalClazz);
     }
   }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == this) {
+      return true;
+    }
+    if (!(other instanceof AvroCoder)) {
+      return false;
+    }
+    AvroCoder<?> that = (AvroCoder<?>) other;
+    return Objects.equals(this.schemaSupplier.get(), that.schemaSupplier.get())
+        && Objects.equals(this.typeDescriptor, that.typeDescriptor);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(schemaSupplier.get(), typeDescriptor);
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
index d628203a48c2..aadf085881af 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
@@ -32,7 +32,7 @@
  * {@link BigInteger}, when scaled (with unlimited precision, aka {@link MathContext#UNLIMITED}),
  * yields the expected {@link BigDecimal}.
  */
-public class BigDecimalCoder extends CustomCoder<BigDecimal> {
+public class BigDecimalCoder extends AtomicCoder<BigDecimal> {
 
   public static BigDecimalCoder of() {
     return INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
index 81c5e9471ad6..c3c7a968d196 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
@@ -29,7 +29,7 @@
 /**
  * A {@link BigEndianIntegerCoder} encodes {@link Integer Integers} in 4 bytes, big-endian.
  */
-public class BigEndianIntegerCoder extends CustomCoder<Integer> {
+public class BigEndianIntegerCoder extends AtomicCoder<Integer> {
 
   public static BigEndianIntegerCoder of() {
     return INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
index 173e91020427..5ef487801540 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
@@ -17,7 +17,6 @@
  */
 package org.apache.beam.sdk.coders;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.EOFException;
@@ -30,9 +29,8 @@
 /**
  * A {@link BigEndianLongCoder} encodes {@link Long}s in 8 bytes, big-endian.
  */
-public class BigEndianLongCoder extends CustomCoder<Long> {
+public class BigEndianLongCoder extends AtomicCoder<Long> {
 
-  @JsonCreator
   public static BigEndianLongCoder of() {
     return INSTANCE;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
index a739da78aa17..6d14d17ee7ce 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
@@ -28,7 +28,7 @@
  * A {@link BigIntegerCoder} encodes a {@link BigInteger} as a byte array containing the big endian
  * two's-complement representation, encoded via {@link ByteArrayCoder}.
  */
-public class BigIntegerCoder extends CustomCoder<BigInteger> {
+public class BigIntegerCoder extends AtomicCoder<BigInteger> {
 
   public static BigIntegerCoder of() {
     return INSTANCE;
@@ -55,7 +55,7 @@ public BigInteger decode(InputStream inStream, Context context)
   }
 
   @Override
-  public void verifyDeterministic() throws NonDeterministicException {
+  public void verifyDeterministic() {
     BYTE_ARRAY_CODER.verifyDeterministic();
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java
index 5a4db24697c5..f49776b28a5f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java
@@ -25,7 +25,7 @@
 /**
  * Coder for {@link BitSet}.
  */
-public class BitSetCoder extends CustomCoder<BitSet> {
+public class BitSetCoder extends AtomicCoder<BitSet> {
   private static final BitSetCoder INSTANCE = new BitSetCoder();
   private static final ByteArrayCoder BYTE_ARRAY_CODER = ByteArrayCoder.of();
 
@@ -53,7 +53,7 @@ public BitSet decode(InputStream inStream, Context context)
   @Override
   public void verifyDeterministic() throws NonDeterministicException {
     verifyDeterministic(
-        "BitSetCoder requires its ByteArrayCoder to be deterministic.", BYTE_ARRAY_CODER);
+        this, "BitSetCoder requires its ByteArrayCoder to be deterministic.", BYTE_ARRAY_CODER);
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
index cba8d49b4c14..28cb627017f6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
@@ -17,7 +17,6 @@
  */
 package org.apache.beam.sdk.coders;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
 import com.google.common.io.ByteStreams;
 import java.io.IOException;
 import java.io.InputStream;
@@ -40,9 +39,8 @@
  * encoded via a {@link VarIntCoder}.</li>
  * </ul>
  */
-public class ByteArrayCoder extends StructuredCoder<byte[]> {
+public class ByteArrayCoder extends AtomicCoder<byte[]> {
 
-  @JsonCreator
   public static ByteArrayCoder of() {
     return INSTANCE;
   }
@@ -117,7 +115,7 @@ public List<? extends Coder<?>> getCoderArguments() {
   }
 
   @Override
-  public void verifyDeterministic() throws NonDeterministicException {}
+  public void verifyDeterministic() {}
 
   /**
    * {@inheritDoc}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
index 1a1be641169a..6e4318e5a3ff 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
@@ -27,7 +27,7 @@
 /**
  * A {@link ByteCoder} encodes {@link Byte} values in 1 byte using Java serialization.
  */
-public class ByteCoder extends CustomCoder<Byte> {
+public class ByteCoder extends AtomicCoder<Byte> {
 
   public static ByteCoder of() {
     return INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
index 2a1d792b7603..4f05c95f0b2c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
@@ -239,11 +239,12 @@ private <T, CoderT extends Coder> Method getInstanceComponentsMethod(Class<Coder
     }
 
     /**
-     * If {@code coderType} is a subclass of {@link Coder} for a specific
-     * type {@code T}, returns {@code T.class}. Otherwise, raises IllegalArgumentException.
+     * If {@code coderType} is a subclass of {@link Coder} for a specific type {@code T}, returns
+     * {@code T.class}. Otherwise, raises IllegalArgumentException.
      */
-    private <T> TypeDescriptor<T> getCodedType(TypeDescriptor<? extends Coder> coderType) {
-      TypeDescriptor<?> coderSupertype = coderType.getSupertype(Coder.class);
+    private <T, CoderT extends Coder> TypeDescriptor<T> getCodedType(
+        TypeDescriptor<CoderT> coderType) {
+      TypeDescriptor<? super CoderT> coderSupertype = coderType.getSupertype(Coder.class);
       ParameterizedType coderIface = (ParameterizedType) coderSupertype.getType();
       @SuppressWarnings("unchecked")
       TypeDescriptor<T> token =
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
index 06e7daef2712..12bc5e8da15f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
@@ -29,7 +29,7 @@
 /**
  * A {@link DoubleCoder} encodes {@link Double} values in 8 bytes using Java serialization.
  */
-public class DoubleCoder extends CustomCoder<Double> {
+public class DoubleCoder extends AtomicCoder<Double> {
 
   public static DoubleCoder of() {
     return INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
index 10a83ef94c34..7b49d1fe31d2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
@@ -29,7 +29,7 @@
  * A {@link Coder} that encodes a joda {@link Duration} as a {@link Long} using the format of
  * {@link VarLongCoder}.
  */
-public class DurationCoder extends CustomCoder<ReadableDuration> {
+public class DurationCoder extends AtomicCoder<ReadableDuration> {
 
   public static DurationCoder of() {
     return INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
index cfd1979545a3..56ed12b01e95 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
@@ -29,7 +29,7 @@
  * A {@link Coder} for joda {@link Instant} that encodes it as a big endian {@link Long}
  * shifted such that lexicographic ordering of the bytes corresponds to chronological order.
  */
-public class InstantCoder extends CustomCoder<Instant> {
+public class InstantCoder extends AtomicCoder<Instant> {
   public static InstantCoder of() {
     return INSTANCE;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
index 8a689f788845..35b7449bcb43 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
@@ -89,8 +89,8 @@ public List<? extends Coder<?>> getCoderArguments() {
 
   @Override
   public void verifyDeterministic() throws NonDeterministicException {
-    verifyDeterministic("Key coder must be deterministic", getKeyCoder());
-    verifyDeterministic("Value coder must be deterministic", getValueCoder());
+    verifyDeterministic(this, "Key coder must be deterministic", getKeyCoder());
+    verifyDeterministic(this, "Value coder must be deterministic", getValueCoder());
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java
index 32467d2cfea5..70bbf9333787 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java
@@ -58,8 +58,7 @@ protected ListCoder(Coder<T> elemCoder) {
    */
   @Override
   public void verifyDeterministic() throws NonDeterministicException {
-    verifyDeterministic(
-        "ListCoder.elemCoder must be deterministic", getElemCoder());
+    verifyDeterministic(this, "ListCoder.elemCoder must be deterministic", getElemCoder());
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
index e2c4e28c1e52..da2bf50d3125 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
@@ -40,7 +40,7 @@
  * @param <K> the type of the keys of the KVs being transcoded
  * @param <V> the type of the values of the KVs being transcoded
  */
-public class MapCoder<K, V> extends CustomCoder<Map<K, V>> {
+public class MapCoder<K, V> extends StructuredCoder<Map<K, V>> {
   /**
    * Produces a MapCoder with the given keyCoder and valueCoder.
    */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
index 747d91c441c4..d1eea9aa2fe4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
@@ -35,7 +35,7 @@
  *
  * @param <T> the type of the values being transcoded
  */
-public class NullableCoder<T> extends CustomCoder<T> {
+public class NullableCoder<T> extends StructuredCoder<T> {
   public static <T> NullableCoder<T> of(Coder<T> valueCoder) {
     if (valueCoder instanceof NullableCoder) {
       return (NullableCoder<T>) valueCoder;
@@ -93,11 +93,11 @@ public List<Coder<T>> getCoderArguments() {
   /**
    * {@code NullableCoder} is deterministic if the nested {@code Coder} is.
    *
-   * {@inheritDoc}
+   * <p>{@inheritDoc}
    */
   @Override
   public void verifyDeterministic() throws NonDeterministicException {
-    verifyDeterministic("Value coder must be deterministic", valueCoder);
+    verifyDeterministic(this, "Value coder must be deterministic", valueCoder);
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
index f0a0969fe0e5..42931ca60671 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
@@ -38,7 +38,7 @@
  * If in a nested context, prefixes the string with an integer length field,
  * encoded via a {@link VarIntCoder}.
  */
-public class StringUtf8Coder extends CustomCoder<String> {
+public class StringUtf8Coder extends AtomicCoder<String> {
 
   public static StringUtf8Coder of() {
     return INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
index 91a46ea69093..9743c4c3f102 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
@@ -26,7 +26,7 @@
  * A {@link Coder} that encodes {@code Integer Integers} as the ASCII bytes of
  * their textual, decimal, representation.
  */
-public class TextualIntegerCoder extends CustomCoder<Integer> {
+public class TextualIntegerCoder extends AtomicCoder<Integer> {
 
   public static TextualIntegerCoder of() {
     return new TextualIntegerCoder();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
index fcc0335b1fcf..30f9c0977a6c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
@@ -30,7 +30,7 @@
  * numbers always take 5 bytes, so {@link BigEndianIntegerCoder} may be preferable for
  * integers that are known to often be large or negative.
  */
-public class VarIntCoder extends CustomCoder<Integer> {
+public class VarIntCoder extends AtomicCoder<Integer> {
 
   public static VarIntCoder of() {
     return INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
index a65fa5ef2710..829bd2020605 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
@@ -17,7 +17,6 @@
  */
 package org.apache.beam.sdk.coders;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
 import java.io.InputStream;
 import java.io.OutputStream;
 import org.apache.beam.sdk.values.TypeDescriptor;
@@ -25,9 +24,8 @@
 /**
  * A {@link Coder} for {@link Void}. Uses zero bytes per {@link Void}.
  */
-public class VoidCoder extends CustomCoder<Void> {
+public class VoidCoder extends AtomicCoder<Void> {
 
-  @JsonCreator
   public static VoidCoder of() {
     return INSTANCE;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 0daf5dc485b3..20fab9b68b62 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -43,8 +43,8 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.GZIPOutputStream;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy.Context;
@@ -938,7 +938,7 @@ public String toString() {
   /**
    * A coder for {@link FileResult} objects.
    */
-  public static final class FileResultCoder extends CustomCoder<FileResult> {
+  public static final class FileResultCoder extends AtomicCoder<FileResult> {
     private static final FileResultCoder INSTANCE = new FileResultCoder();
     private final NullableCoder<String> stringCoder = NullableCoder.of(StringUtf8Coder.of());
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
index 5432f0975e3e..b05f2233156a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
@@ -34,6 +34,7 @@
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 import java.util.PriorityQueue;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
@@ -761,14 +762,29 @@ public void registerByteSizeObserver(
       }
     }
 
+    @Override
+    public boolean equals(Object other) {
+      if (other == this) {
+        return true;
+      }
+      if (!(other instanceof QuantileStateCoder)) {
+        return false;
+      }
+      QuantileStateCoder<?, ?> that = (QuantileStateCoder<?, ?>) other;
+      return Objects.equals(this.elementCoder, that.elementCoder)
+          && Objects.equals(this.compareFn, that.compareFn);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(elementCoder, compareFn);
+    }
+
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
+      verifyDeterministic(this, "QuantileState.ElementCoder must be deterministic", elementCoder);
       verifyDeterministic(
-          "QuantileState.ElementCoder must be deterministic",
-          elementCoder);
-      verifyDeterministic(
-          "QuantileState.ElementListCoder must be deterministic",
-          elementListCoder);
+          this, "QuantileState.ElementListCoder must be deterministic", elementListCoder);
     }
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 666db3b80a65..b9cdbd5f407f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -27,6 +27,7 @@
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -35,10 +36,10 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.DelegateCoder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -522,7 +523,7 @@ private void set(V value) {
   /**
    * A {@link Coder} for a {@link Holder}.
    */
-  private static class HolderCoder<V> extends CustomCoder<Holder<V>> {
+  private static class HolderCoder<V> extends StructuredCoder<Holder<V>> {
 
     private Coder<V> valueCoder;
 
@@ -551,6 +552,11 @@ public Holder<V> decode(InputStream inStream, Context context)
       }
     }
 
+    @Override
+    public List<? extends Coder<?>> getCoderArguments() {
+      return Collections.singletonList(valueCoder);
+    }
+
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
       valueCoder.verifyDeterministic();
@@ -1954,7 +1960,7 @@ public static <InputT, AccumT> InputOrAccum<InputT, AccumT> accum(AccumT aggr) {
       }
 
       private static class InputOrAccumCoder<InputT, AccumT>
-          extends CustomCoder<InputOrAccum<InputT, AccumT>> {
+          extends StructuredCoder<InputOrAccum<InputT, AccumT>> {
 
         private final Coder<InputT> inputCoder;
         private final Coder<AccumT> accumCoder;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
index cc02dcfbb4f1..d4c97bc3c4c9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
@@ -35,7 +35,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
@@ -524,7 +524,7 @@ public void remove() {
     }
   }
 
-  private static class ComposedAccumulatorCoder extends CustomCoder<Object[]> {
+  private static class ComposedAccumulatorCoder extends StructuredCoder<Object[]> {
     private List<Coder<Object>> coders;
     private int codersCount;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
index 78a6cd13fe72..753e14c84c96 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
@@ -23,10 +23,10 @@
 import java.io.OutputStream;
 import java.io.UTFDataFormatException;
 import java.util.Iterator;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.util.VarInt;
@@ -167,7 +167,7 @@ public Long extractOutput(long[] accumulator) {
     @Override
     public Coder<long[]> getAccumulatorCoder(CoderRegistry registry,
                                              Coder<T> inputCoder) {
-      return new CustomCoder<long[]>() {
+      return new AtomicCoder<long[]>() {
         @Override
         public void encode(long[] value, OutputStream outStream, Context context)
             throws IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
index a6808cf15013..a309954982f1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
@@ -22,11 +22,11 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Objects;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.DoubleCoder;
 import org.apache.beam.sdk.transforms.Combine.AccumulatingCombineFn.Accumulator;
 
@@ -180,9 +180,8 @@ public String toString() {
     }
   }
 
-  static class CountSumCoder<NumT extends Number>
-  extends CustomCoder<CountSum<NumT>> {
-     private static final Coder<Long> LONG_CODER = BigEndianLongCoder.of();
+  static class CountSumCoder<NumT extends Number> extends AtomicCoder<CountSum<NumT>> {
+    private static final Coder<Long> LONG_CODER = BigEndianLongCoder.of();
      private static final Coder<Double> DOUBLE_CODER = DoubleCoder.of();
 
      @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
index e42c0b2dd9ea..9d5db74a7c52 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
@@ -27,6 +27,7 @@
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Objects;
 import java.util.PriorityQueue;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
@@ -551,8 +552,7 @@ public BoundedHeap<T, ComparatorT> decode(InputStream inStream, Coder.Context co
 
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
-      verifyDeterministic(
-          "HeapCoder requires a deterministic list coder", listCoder);
+      verifyDeterministic(this, "HeapCoder requires a deterministic list coder", listCoder);
     }
 
     @Override
@@ -568,5 +568,24 @@ public void registerByteSizeObserver(
             throws Exception {
       listCoder.registerByteSizeObserver(value.asList(), observer, context);
     }
+
+    @Override
+    public boolean equals(Object other) {
+      if (other == this) {
+        return true;
+      }
+      if (!(other instanceof BoundedHeapCoder)) {
+        return false;
+      }
+      BoundedHeapCoder<?, ?> that = (BoundedHeapCoder<?, ?>) other;
+      return Objects.equals(this.compareFn, that.compareFn)
+          && Objects.equals(this.listCoder, that.listCoder)
+          && this.maximumSize == that.maximumSize;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(compareFn, listCoder, maximumSize);
+    }
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
index 02e1185998d7..e9a3571216ff 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
@@ -300,7 +300,7 @@ public int hashCode() {
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
       verifyDeterministic(
-          "CoGbkResult requires the union coder to be deterministic", unionCoder);
+          this, "CoGbkResult requires the union coder to be deterministic", unionCoder);
     }
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
index f411cd1333b4..4a2a2867bd22 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
@@ -23,14 +23,14 @@
 import java.util.List;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.util.VarInt;
 import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
 
 /**
  * A UnionCoder encodes RawUnionValues.
  */
-public class UnionCoder extends CustomCoder<RawUnionValue> {
+public class UnionCoder extends StructuredCoder<RawUnionValue> {
   // TODO: Think about how to integrate this with a schema object (i.e.
   // a tuple of tuple tags).
   /**
@@ -134,7 +134,6 @@ private UnionCoder(List<Coder<?>> elementCoders) {
   @Override
   public void verifyDeterministic() throws NonDeterministicException {
     verifyDeterministic(
-        "UnionCoder is only deterministic if all element coders are",
-        elementCoders);
+        this, "UnionCoder is only deterministic if all element coders are", elementCoders);
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
index faf3ca93612c..79ce2f5bf24a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
@@ -26,9 +26,9 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Objects;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
 import org.apache.beam.sdk.transforms.GroupByKey;
@@ -307,7 +307,7 @@ public String toString() {
   /**
    * A Coder for encoding PaneInfo instances.
    */
-  public static class PaneInfoCoder extends CustomCoder<PaneInfo> {
+  public static class PaneInfoCoder extends AtomicCoder<PaneInfo> {
     private enum Encoding {
       FIRST,
       ONE_INDEX,
@@ -340,6 +340,12 @@ private Encoding chooseEncoding(PaneInfo value) {
 
     public static final PaneInfoCoder INSTANCE = new PaneInfoCoder();
 
+    public static PaneInfoCoder of() {
+      return INSTANCE;
+    }
+
+    private PaneInfoCoder() {}
+
     @Override
     public void encode(PaneInfo value, final OutputStream outStream, Coder.Context context)
         throws CoderException, IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
index b0e9b5c50254..b646bf6b7107 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
@@ -21,15 +21,17 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.BitSet;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 
 /**
  * Coder for the BitSet used to track child-trigger finished states.
+ *
+ * @deprecated use {@link org.apache.beam.sdk.coders.BitSetCoder} instead
  */
 @Deprecated
-public class BitSetCoder extends CustomCoder<BitSet> {
+public class BitSetCoder extends AtomicCoder<BitSet> {
 
   private static final BitSetCoder INSTANCE = new BitSetCoder();
   private static final ByteArrayCoder BYTE_ARRAY_CODER = ByteArrayCoder.of();
@@ -54,8 +56,7 @@ public BitSet decode(InputStream inStream, Context context)
 
   @Override
   public void verifyDeterministic() throws NonDeterministicException {
-    verifyDeterministic(
-        "BitSetCoder requires its ByteArrayCoder to be deterministic.", BYTE_ARRAY_CODER);
+    BYTE_ARRAY_CODER.verifyDeterministic();
   }
 }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
index 13e499ddcb5f..1e725500e5ff 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
@@ -663,11 +663,9 @@ public WindowedValue<T> decode(InputStream inStream, Context context)
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
       verifyDeterministic(
-          "FullWindowedValueCoder requires a deterministic valueCoder",
-          valueCoder);
+          this, "FullWindowedValueCoder requires a deterministic valueCoder", valueCoder);
       verifyDeterministic(
-          "FullWindowedValueCoder requires a deterministic windowCoder",
-          windowCoder);
+          this, "FullWindowedValueCoder requires a deterministic windowCoder", windowCoder);
     }
 
     @Override
@@ -728,6 +726,7 @@ public WindowedValue<T> decode(InputStream inStream, Context context)
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
       verifyDeterministic(
+          this,
           "ValueOnlyWindowedValueCoder requires a deterministic valueCoder",
           valueCoder);
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
index cde9a40d083d..a9f39298b123 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
@@ -23,11 +23,12 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.InstantCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.joda.time.Instant;
 
@@ -85,7 +86,7 @@ public String toString() {
   /////////////////////////////////////////////////////////////////////////////
 
   /** A {@link Coder} for {@link TimestampedValue}. */
-  public static class TimestampedValueCoder<T> extends CustomCoder<TimestampedValue<T>> {
+  public static class TimestampedValueCoder<T> extends StructuredCoder<TimestampedValue<T>> {
 
     private final Coder<T> valueCoder;
 
@@ -119,8 +120,7 @@ public TimestampedValue<T> decode(InputStream inStream, Context context)
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
       verifyDeterministic(
-          "TimestampedValueCoder requires a deterministic valueCoder",
-          valueCoder);
+          this, "TimestampedValueCoder requires a deterministic valueCoder", valueCoder);
     }
 
     @Override
@@ -141,6 +141,11 @@ public TypeDescriptor<TimestampedValue<T>> getEncodedTypeDescriptor() {
       return new TypeDescriptor<TimestampedValue<T>>() {}.where(
           new TypeParameter<T>() {}, valueCoder.getEncodedTypeDescriptor());
     }
+
+    @Override
+    public List<? extends Coder<?>> getComponents() {
+      return Collections.singletonList(valueCoder);
+    }
   }
 
   /////////////////////////////////////////////////////////////////////////////
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
index 1fd356b9cf56..3ecbaa2072ad 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
@@ -24,8 +24,8 @@
 import java.io.OutputStream;
 import java.util.List;
 import javax.annotation.Nullable;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.InstantCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.joda.time.Instant;
@@ -56,7 +56,7 @@ public static <T> ValueInSingleWindow<T> of(
   }
 
   /** A coder for {@link ValueInSingleWindow}. */
-  public static class Coder<T> extends CustomCoder<ValueInSingleWindow<T>> {
+  public static class Coder<T> extends StructuredCoder<ValueInSingleWindow<T>> {
     private final org.apache.beam.sdk.coders.Coder<T> valueCoder;
     private final org.apache.beam.sdk.coders.Coder<BoundedWindow> windowCoder;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
index 0d92f4075d55..3f057e16cf2d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
@@ -27,7 +27,7 @@
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.transforms.DoFn;
 
 /**
@@ -85,7 +85,7 @@ public int hashCode() {
    * A {@link Coder} for {@code ValueWithRecordId}, using a wrapped value {@code Coder}.
    */
   public static class ValueWithRecordIdCoder<ValueT>
-      extends CustomCoder<ValueWithRecordId<ValueT>> {
+      extends StructuredCoder<ValueWithRecordId<ValueT>> {
     public static <ValueT> ValueWithRecordIdCoder<ValueT> of(Coder<ValueT> valueCoder) {
       return new ValueWithRecordIdCoder<>(valueCoder);
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
index fe21a1c2bca5..5107355e0ea8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
@@ -99,7 +99,7 @@ public void testRegisterInstantiatedCoder() throws Exception {
   }
 
   @SuppressWarnings("rawtypes") // this class exists to fail a test because of its rawtypes
-  private class MyListCoder extends CustomCoder<List> {
+  private class MyListCoder extends AtomicCoder<List> {
     @Override
     public void encode(List value, OutputStream outStream, Context context)
         throws CoderException, IOException {
@@ -441,7 +441,7 @@ private static class MyGenericClass<FooT, BazT> { }
 
   private static class MyValue { }
 
-  private static class MyValueCoder extends CustomCoder<MyValue> {
+  private static class MyValueCoder extends AtomicCoder<MyValue> {
 
     private static final MyValueCoder INSTANCE = new MyValueCoder();
     private static final TypeDescriptor<MyValue> TYPE_DESCRIPTOR = TypeDescriptor.of(MyValue.class);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java
index 5ff272fd51fc..8aeb22a27cf5 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java
@@ -24,8 +24,6 @@
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.Arrays;
 import java.util.Collections;
@@ -108,29 +106,6 @@ public void testSerializable() throws Exception {
   private static final String TEST_ENCODING_ID = "test-encoding-id";
   private static final String TEST_ALLOWED_ENCODING = "test-allowed-encoding";
 
-  private static class TestAllowedEncodingsCoder extends CustomCoder<Integer> {
-
-    @Override
-    public void encode(Integer value, OutputStream outstream, Context context) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Integer decode(InputStream instream, Context context) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void verifyDeterministic() {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public List<? extends Coder<?>> getCoderArguments() {
-      return Collections.emptyList();
-    }
-  }
-
   @Test
   public void testCoderEquals() throws Exception {
     DelegateCoder.CodingFunction<Integer, Integer> identityFn =
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
index 894d2d194236..c0a4bed84a42 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
@@ -169,7 +169,7 @@ public void testEncodedTypeDescriptor() throws Exception {
     assertThat(TEST_CODER.getEncodedTypeDescriptor(), equalTo(TypeDescriptor.of(String.class)));
   }
 
-  private static class EntireStreamExpectingCoder extends CustomCoder<String> {
+  private static class EntireStreamExpectingCoder extends AtomicCoder<String> {
     @Override
     public void encode(
         String value, OutputStream outStream, Context context) throws IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java
index f337f36215a9..164d22131446 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java
@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CustomCoder;
@@ -47,7 +48,7 @@ public void testGoodCoderIsDeterministic() throws Exception {
   }
 
   /** A coder that says it is not deterministic but actually is. */
-  public static class NonDeterministicCoder extends CustomCoder<String> {
+  public static class NonDeterministicCoder extends AtomicCoder<String> {
     @Override
     public void encode(String value, OutputStream outStream, Context context)
         throws CoderException, IOException {
@@ -59,17 +60,23 @@ public String decode(InputStream inStream, Context context)
         throws CoderException, IOException {
       return StringUtf8Coder.of().decode(inStream, context);
     }
+
+    public void verifyDeterministic() throws NonDeterministicException {
+      throw new NonDeterministicException(this, "Not Deterministic");
+    }
   }
 
   @Test
   public void testNonDeterministicCoder() throws Exception {
     try {
       CoderProperties.coderDeterministic(new NonDeterministicCoder(), "TestData", "TestData");
-      fail("Expected AssertionError");
     } catch (AssertionError error) {
       assertThat(error.getMessage(),
           CoreMatchers.containsString("Expected that the coder is deterministic"));
+      // success!
+      return;
     }
+    fail("Expected AssertionError");
   }
 
   @Test
@@ -84,7 +91,7 @@ public void testPassingInNonEqualValuesWithDeterministicCoder() throws Exception
   }
 
   /** A coder that is non-deterministic because it adds a string to the value. */
-  private static class BadDeterminsticCoder extends CustomCoder<String> {
+  private static class BadDeterminsticCoder extends AtomicCoder<String> {
     public BadDeterminsticCoder() {
     }
 
@@ -141,6 +148,17 @@ public String decode(InputStream inStream, Context context)
       String decodedValue = StringUtf8Coder.of().decode(inStream, context);
       return decodedValue.substring(0, decodedValue.length() - changedState);
     }
+
+    @Override
+    public boolean equals(Object other) {
+      return other instanceof StateChangingSerializingCoder
+          && ((StateChangingSerializingCoder) other).changedState == this.changedState;
+    }
+
+    @Override
+    public int hashCode() {
+      return changedState;
+    }
   }
 
   @Test
@@ -175,6 +193,17 @@ public String decode(InputStream inStream, Context context)
         throws CoderException, IOException {
       return StringUtf8Coder.of().decode(inStream, context);
     }
+
+    @Override
+    public boolean equals(Object other) {
+      return (other instanceof ForgetfulSerializingCoder)
+          && ((ForgetfulSerializingCoder) other).lostState == lostState;
+    }
+
+    @Override
+    public int hashCode() {
+      return lostState;
+    }
   }
 
   @Test
@@ -185,7 +214,7 @@ public void testBadCoderThatDependsOnStateThatIsLost() throws Exception {
   }
 
   /** A coder which closes the underlying stream during encoding and decoding. */
-  public static class ClosingCoder extends CustomCoder<String> {
+  public static class ClosingCoder extends AtomicCoder<String> {
     @Override
     public void encode(String value, OutputStream outStream, Context context) throws IOException {
       outStream.close();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
index cfe743664a9b..2ef892c0e822 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
@@ -35,8 +35,8 @@
 import java.util.Collections;
 import java.util.regex.Pattern;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.io.GenerateSequence;
@@ -88,7 +88,7 @@ public int hashCode() {
     }
   }
 
-  private static class NotSerializableObjectCoder extends CustomCoder<NotSerializableObject> {
+  private static class NotSerializableObjectCoder extends AtomicCoder<NotSerializableObject> {
     private NotSerializableObjectCoder() { }
     private static final NotSerializableObjectCoder INSTANCE = new NotSerializableObjectCoder();
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
index ddc92d610205..db5ff2e3049b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
@@ -29,8 +29,8 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.values.KV;
 import org.hamcrest.Matchers;
@@ -151,7 +151,7 @@ private static class NotSerializableClass {
     }
   }
 
-  private static class NotSerializableClassCoder extends CustomCoder<NotSerializableClass> {
+  private static class NotSerializableClassCoder extends AtomicCoder<NotSerializableClass> {
     @Override
     public void encode(NotSerializableClass value, OutputStream outStream, Coder.Context context) {
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java
index 38a2fa2375bc..546683b44e36 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java
@@ -24,8 +24,8 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Collections;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.util.SerializableUtils;
@@ -72,12 +72,12 @@ public void unencodableWindowFails() {
         Collections.<BoundedWindow>singleton(window));
   }
 
-  private static class FailingCoder extends CustomCoder<BoundedWindow>  {
+  private static class FailingCoder extends AtomicCoder<BoundedWindow> {
     @Override
     public void encode(
         BoundedWindow value, OutputStream outStream, Context context)
         throws CoderException, IOException {
-      throw new CoderException("Test Enccode Exception");
+      throw new CoderException("Test Encode Exception");
     }
 
     @Override
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
index 9250dfa167a3..62edac9b55d9 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
@@ -28,10 +28,10 @@
 import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -326,7 +326,7 @@ static UserString of(String strValue) {
     }
   }
 
-  private static class UserStringCoder extends CustomCoder<UserString> {
+  private static class UserStringCoder extends AtomicCoder<UserString> {
     public static UserStringCoder of() {
       return INSTANCE;
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
index 82c25040e207..12619e03f8ac 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
@@ -41,12 +41,12 @@
 import java.util.Objects;
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.DoubleCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
@@ -874,7 +874,7 @@ public Coder<CountSum> getAccumulatorCoder(
     /**
      * A {@link Coder} for {@link CountSum}.
      */
-    private class CountSumCoder extends CustomCoder<CountSum> {
+    private class CountSumCoder extends AtomicCoder<CountSum> {
       @Override
       public void encode(CountSum value, OutputStream outStream,
           Context context) throws CoderException, IOException {
@@ -923,7 +923,7 @@ public Accumulator(String value) {
       }
 
       public static Coder<Accumulator> getCoder() {
-        return new CustomCoder<Accumulator>() {
+        return new AtomicCoder<Accumulator>() {
           @Override
           public void encode(Accumulator accumulator, OutputStream outStream, Coder.Context context)
               throws CoderException, IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
index 89a1f332704a..76f61b32dc67 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
@@ -36,10 +36,10 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
@@ -132,7 +132,7 @@ static class Record implements Serializable {
   static class Record2 extends Record {
   }
 
-  private static class RecordCoder extends CustomCoder<Record> {
+  private static class RecordCoder extends AtomicCoder<Record> {
     @Override
     public void encode(Record value, OutputStream outStream, Context context)
         throws CoderException, IOException {}
@@ -201,7 +201,7 @@ public boolean equals(Object o) {
       return myString.equals(((UnserializableRecord) o).myString);
     }
 
-    static class UnserializableRecordCoder extends CustomCoder<UnserializableRecord> {
+    static class UnserializableRecordCoder extends AtomicCoder<UnserializableRecord> {
       private final Coder<String> stringCoder = StringUtf8Coder.of();
 
       @Override
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
index 9cb642a77ff4..6982e01116bc 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
@@ -38,9 +38,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.MapCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -453,7 +453,7 @@ public int hashCode() {
   /**
    * Deterministic {@link Coder} for {@link BadEqualityKey}.
    */
-  static class DeterministicKeyCoder extends CustomCoder<BadEqualityKey> {
+  static class DeterministicKeyCoder extends AtomicCoder<BadEqualityKey> {
 
     public static DeterministicKeyCoder of() {
       return INSTANCE;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index ffdf3d0506a8..073957f30001 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -55,9 +55,9 @@
 import java.util.Map;
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.SetCoder;
@@ -976,7 +976,7 @@ public void processElement(ProcessContext c) {
 
   private static class TestDummy { }
 
-  private static class TestDummyCoder extends CustomCoder<TestDummy> {
+  private static class TestDummyCoder extends AtomicCoder<TestDummy> {
     private TestDummyCoder() { }
     private static final TestDummyCoder INSTANCE = new TestDummyCoder();
 
@@ -1085,7 +1085,7 @@ public String toString() {
     }
   }
 
-  private static class MyIntegerCoder extends CustomCoder<MyInteger> {
+  private static class MyIntegerCoder extends AtomicCoder<MyInteger> {
     private static final MyIntegerCoder INSTANCE = new MyIntegerCoder();
 
     private final VarIntCoder delegate = VarIntCoder.of();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
index e72c5401ea81..84f3d69dd10a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
@@ -39,9 +39,9 @@
 import java.util.NoSuchElementException;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -505,7 +505,7 @@ public void processElement(ProcessContext c) {
     pipeline.run();
   }
 
-  private static class NonDeterministicStringCoder extends CustomCoder<String> {
+  private static class NonDeterministicStringCoder extends AtomicCoder<String> {
     @Override
     public void encode(String value, OutputStream outStream, Coder.Context context)
         throws CoderException, IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index 19b60925c361..f26dd59ba9cc 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -35,8 +35,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
@@ -270,7 +270,7 @@ private static class SomeRestriction {}
   private abstract static class SomeRestrictionTracker
       implements RestrictionTracker<SomeRestriction> {}
 
-  private static class SomeRestrictionCoder extends CustomCoder<SomeRestriction> {
+  private static class SomeRestrictionCoder extends AtomicCoder<SomeRestriction> {
     public static SomeRestrictionCoder of() {
       return new SomeRestrictionCoder();
     }
@@ -392,7 +392,7 @@ public RestrictionWithDefaultTracker checkpoint() {
     public void checkDone() throws IllegalStateException {}
   }
 
-  private static class CoderForDefaultTracker extends CustomCoder<RestrictionWithDefaultTracker> {
+  private static class CoderForDefaultTracker extends AtomicCoder<RestrictionWithDefaultTracker> {
     public static CoderForDefaultTracker of() {
       return new CoderForDefaultTracker();
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java
index 0db53553d602..7230a8bbaae4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java
@@ -24,10 +24,10 @@
 
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.testing.CoderPropertiesTest.ClosingCoder;
 import org.junit.Rule;
 import org.junit.Test;
@@ -44,7 +44,7 @@ public class CoderUtilsTest {
   @Rule
   public transient ExpectedException expectedException = ExpectedException.none();
 
-  static class TestCoder extends CustomCoder<Integer> {
+  static class TestCoder extends AtomicCoder<Integer> {
     public static TestCoder of() {
       return new TestCoder();
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
index 2d5baf24a16b..6ba1d4a8af70 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
@@ -25,9 +25,9 @@
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.List;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -85,7 +85,7 @@ public void testSerializationError() {
   }
 
   /** A {@link Coder} that is not serializable by Java. */
-  private static class UnserializableCoderByJava extends CustomCoder<Object> {
+  private static class UnserializableCoderByJava extends AtomicCoder<Object> {
     private final Object unserializableField = new Object();
 
     @Override
diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
index d4e6f63f116c..0781cf1fdac8 100644
--- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
+++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
@@ -22,9 +22,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.util.VarInt;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
@@ -34,7 +34,7 @@
  * <p>When this code is used in a nested {@link Coder.Context}, the serialized {@link ByteString}
  * objects are first delimited by their size.
  */
-public class ByteStringCoder extends CustomCoder<ByteString> {
+public class ByteStringCoder extends AtomicCoder<ByteString> {
 
   public static ByteStringCoder of() {
     return INSTANCE;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
index 26b4b5639ccb..f04e9b9ae7f2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
@@ -25,7 +25,7 @@
 import java.util.Arrays;
 import java.util.List;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 
 
@@ -34,11 +34,14 @@
  */
 @VisibleForTesting
 class ShardedKeyCoder<KeyT>
-    extends CustomCoder<ShardedKey<KeyT>> {
+    extends StructuredCoder<ShardedKey<KeyT>> {
   public static <KeyT> ShardedKeyCoder<KeyT> of(Coder<KeyT> keyCoder) {
     return new ShardedKeyCoder<>(keyCoder);
   }
 
+  private final Coder<KeyT> keyCoder;
+  private final VarIntCoder shardNumberCoder;
+
   protected ShardedKeyCoder(Coder<KeyT> keyCoder) {
     this.keyCoder = keyCoder;
     this.shardNumberCoder = VarIntCoder.of();
@@ -68,7 +71,4 @@ public ShardedKey<KeyT> decode(InputStream inStream, Context context)
   public void verifyDeterministic() throws NonDeterministicException {
     keyCoder.verifyDeterministic();
   }
-
-  Coder<KeyT> keyCoder;
-  VarIntCoder shardNumberCoder;
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
index 8a06d1397849..3059e2aecde2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
@@ -21,12 +21,12 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 
 /** A coder for {@link TableDestination} objects. */
-public class TableDestinationCoder extends CustomCoder<TableDestination> {
+public class TableDestinationCoder extends AtomicCoder<TableDestination> {
   private static final TableDestinationCoder INSTANCE = new TableDestinationCoder();
   private static final StringUtf8Coder stringCoder = StringUtf8Coder.of();
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
index c3e48a4db9ee..2b1988a2bda7 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
@@ -22,15 +22,15 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 
 /**
  * Defines a coder for {@link TableRowInfo} objects.
  */
 @VisibleForTesting
-class TableRowInfoCoder extends CustomCoder<TableRowInfo> {
+class TableRowInfoCoder extends AtomicCoder<TableRowInfo> {
   private static final TableRowInfoCoder INSTANCE = new TableRowInfoCoder();
 
   public static TableRowInfoCoder of() {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
index a1ca41b24431..7ca8958e01de 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
@@ -23,15 +23,15 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
  * A {@link Coder} that encodes BigQuery {@link TableRow} objects in their native JSON format.
  */
-public class TableRowJsonCoder extends CustomCoder<TableRow> {
+public class TableRowJsonCoder extends AtomicCoder<TableRow> {
 
   public static TableRowJsonCoder of() {
     return INSTANCE;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
index 70aa135aab27..b8960835e440 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
@@ -26,12 +26,14 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -71,7 +73,7 @@ public Result(String filename, Long fileByteSize, DestinationT destination) {
   }
 
   /** a coder for the {@link Result} class. */
-  public static class ResultCoder<DestinationT> extends CustomCoder<Result<DestinationT>> {
+  public static class ResultCoder<DestinationT> extends StructuredCoder<Result<DestinationT>> {
     private static final StringUtf8Coder stringCoder = StringUtf8Coder.of();
     private static final VarLongCoder longCoder = VarLongCoder.of();
     private final Coder<DestinationT> destinationCoder;
@@ -104,6 +106,11 @@ public Result<DestinationT> decode(InputStream inStream, Context context) throws
       return new Result<>(filename, fileByteSize, destination);
     }
 
+    @Override
+    public List<? extends Coder<?>> getCoderArguments() {
+      return Collections.singletonList(destinationCoder);
+    }
+
     @Override
     public void verifyDeterministic() {}
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
index 031d9a0ab14e..9f04a6c3fe07 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
@@ -31,11 +31,11 @@
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.MapCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
@@ -100,7 +100,7 @@ public class PubsubUnboundedSink extends PTransform<PCollection<PubsubMessage>,
   /**
    * Coder for conveying outgoing messages between internal stages.
    */
-  private static class OutgoingMessageCoder extends CustomCoder<OutgoingMessage> {
+  private static class OutgoingMessageCoder extends AtomicCoder<OutgoingMessage> {
     private static final NullableCoder<String> RECORD_ID_CODER =
         NullableCoder.of(StringUtf8Coder.of());
     private static final NullableCoder<Map<String, String>> ATTRIBUTES_CODER =
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index c2cbe73f484b..c16b8fb70eb2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -46,8 +46,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -118,7 +118,7 @@ public class PubsubUnboundedSource extends PTransform<PBegin, PCollection<Pubsub
   /**
    * Coder for checkpoints.
    */
-  private static final PubsubCheckpointCoder CHECKPOINT_CODER = new PubsubCheckpointCoder();
+  private static final PubsubCheckpointCoder<?> CHECKPOINT_CODER = PubsubCheckpointCoder.of();
 
   /**
    * Maximum number of messages per pull.
@@ -357,11 +357,17 @@ public void nackAll(PubsubReader reader) throws IOException {
   }
 
   /** The coder for our checkpoints. */
-  private static class PubsubCheckpointCoder extends CustomCoder<PubsubCheckpoint> {
+  private static class PubsubCheckpointCoder<T> extends AtomicCoder<PubsubCheckpoint> {
     private static final Coder<String> SUBSCRIPTION_PATH_CODER =
         NullableCoder.of(StringUtf8Coder.of());
     private static final Coder<List<String>> LIST_CODER = ListCoder.of(StringUtf8Coder.of());
 
+    public static <T> PubsubCheckpointCoder<T> of() {
+      return new PubsubCheckpointCoder<>();
+    }
+
+    private PubsubCheckpointCoder() {}
+
     @Override
     public void encode(PubsubCheckpoint value, OutputStream outStream, Context context)
         throws IOException {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index a3b21eebd802..6c15f8736377 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -73,10 +73,10 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -720,7 +720,7 @@ public int hashCode() {
   /**
    * Coder for @link{PartitionedGlobalWindow}.
    */
-  private static class PartitionedGlobalWindowCoder extends CustomCoder<PartitionedGlobalWindow> {
+  private static class PartitionedGlobalWindowCoder extends AtomicCoder<PartitionedGlobalWindow> {
     @Override
     public void encode(PartitionedGlobalWindow window, OutputStream outStream, Context context)
         throws IOException, CoderException {
diff --git a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
index 9589fb144eca..15877b03ad93 100644
--- a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
+++ b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
@@ -24,6 +24,7 @@
 import java.io.OutputStream;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CustomCoder;
@@ -92,4 +93,21 @@ public void verifyDeterministic() throws NonDeterministicException {
         "Hadoop Writable may be non-deterministic.");
   }
 
+  @Override
+  public boolean equals(Object other) {
+    if (other == this) {
+      return true;
+    }
+    if (!(other instanceof WritableCoder)) {
+      return false;
+    }
+    WritableCoder<?> that = (WritableCoder<?>) other;
+    return Objects.equals(this.type, that.type);
+  }
+
+  @Override
+  public int hashCode() {
+    return type.hashCode();
+  }
+
 }
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
index 35a88638e648..7cc043ce34db 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
@@ -21,8 +21,8 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
@@ -34,7 +34,7 @@
  * A {@link Coder} that serializes and deserializes the {@link Mutation} objects using {@link
  * ProtobufUtil}.
  */
-class HBaseMutationCoder extends CustomCoder<Mutation> implements Serializable {
+class HBaseMutationCoder extends AtomicCoder<Mutation> implements Serializable {
   private static final HBaseMutationCoder INSTANCE = new HBaseMutationCoder();
 
   private HBaseMutationCoder() {}
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
index 0004d03bd12f..24a5f7f090aa 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
@@ -21,8 +21,8 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
@@ -31,7 +31,7 @@
  * A {@link Coder} that serializes and deserializes the {@link Result} objects using {@link
  * ProtobufUtil}.
  */
-class HBaseResultCoder extends CustomCoder<Result> implements Serializable {
+class HBaseResultCoder extends AtomicCoder<Result> implements Serializable {
   private static final HBaseResultCoder INSTANCE = new HBaseResultCoder();
 
   private HBaseResultCoder() {}
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index f4de76a56b47..e67645529a1f 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -55,11 +55,11 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.io.Read.Unbounded;
@@ -1595,11 +1595,11 @@ public void populateDisplayData(DisplayData.Builder builder) {
     }
   }
 
-  private static class NullOnlyCoder<T> extends CustomCoder<T> {
+  private static class NullOnlyCoder<T> extends AtomicCoder<T> {
     @Override
     public void encode(T value, OutputStream outStream, Context context) {
       checkArgument(value == null, "Can only encode nulls");
-      // Encode as the empty string.
+      // Encode as no bytes.
     }
 
     @Override
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
index 160e8ce990ac..0f5dc4cd19f5 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
@@ -23,9 +23,9 @@
 import java.util.List;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.values.KV;
@@ -33,7 +33,7 @@
 /**
  * {@link Coder} for {@link KafkaRecord}.
  */
-public class KafkaRecordCoder<K, V> extends CustomCoder<KafkaRecord<K, V>> {
+public class KafkaRecordCoder<K, V> extends StructuredCoder<KafkaRecord<K, V>> {
 
   private static final StringUtf8Coder stringCoder = StringUtf8Coder.of();
   private static final VarLongCoder longCoder = VarLongCoder.of();
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
index 4da2b05dfd04..77fe127bcba6 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
@@ -21,9 +21,9 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
+import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarLongCoder;
@@ -32,7 +32,7 @@
 /**
  * A {@link Coder} for {@link KinesisRecord}.
  */
-class KinesisRecordCoder extends CustomCoder<KinesisRecord> {
+class KinesisRecordCoder extends AtomicCoder<KinesisRecord> {
     private static final StringUtf8Coder STRING_CODER = StringUtf8Coder.of();
     private static final ByteArrayCoder BYTE_ARRAY_CODER = ByteArrayCoder.of();
     private static final InstantCoder INSTANT_CODER = InstantCoder.of();
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
index 812bc70219ca..75a461924907 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
@@ -23,6 +23,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Objects;
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Marshaller;
@@ -139,6 +140,23 @@ public TypeDescriptor<T> getEncodedTypeDescriptor() {
     return TypeDescriptor.of(jaxbClass);
   }
 
+  @Override
+  public boolean equals(Object other) {
+    if (other == this) {
+      return true;
+    }
+    if (!(other instanceof JAXBCoder)) {
+      return false;
+    }
+    JAXBCoder<?> that = (JAXBCoder<?>) other;
+    return Objects.equals(this.jaxbClass, that.jaxbClass);
+  }
+
+  @Override
+  public int hashCode() {
+    return jaxbClass.hashCode();
+  }
+
   private static class CloseIgnoringInputStream extends FilterInputStream {
 
     protected CloseIgnoringInputStream(InputStream in) {
diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
index 940d596a0b96..2b4503ac5bca 100644
--- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
+++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
@@ -33,7 +33,7 @@
 import javax.xml.bind.annotation.XmlRootElement;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.testing.CoderProperties;
@@ -171,7 +171,7 @@ public void run() {
   /**
    * A coder that surrounds the value with two values, to demonstrate nesting.
    */
-  private static class TestCoder extends CustomCoder<TestType> {
+  private static class TestCoder extends StructuredCoder<TestType> {
     private final JAXBCoder<TestType> jaxbCoder;
     public TestCoder(JAXBCoder<TestType> jaxbCoder) {
       this.jaxbCoder = jaxbCoder;

From c7f3e3c3a00423dfcaca6e5b53ad4f84533a56f6 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 5 May 2017 10:13:41 -0700
Subject: [PATCH 082/387] Make CustomCoder extend Coder directly

CustomCoder in general consists of configuration beyond component
coders, and should not extend, for example, equality methods of
StrucutredCoder.
---
 .../apache/beam/sdk/coders/CustomCoder.java   | 50 ++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
index 87bd531f8b3a..f33e2104d3d7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
@@ -17,6 +17,9 @@
  */
 package org.apache.beam.sdk.coders;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
@@ -33,8 +36,53 @@
  *
  * @param <T> the type of elements handled by this coder
  */
-public abstract class CustomCoder<T> extends StructuredCoder<T>
+public abstract class CustomCoder<T> extends Coder<T>
     implements Serializable {
+
+  @Override
+  public void encode(T value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Coder.Context.NESTED);
+  }
+
+  @Deprecated
+  @Override
+  public void encodeOuter(T value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Coder.Context.OUTER);
+  }
+
+  @Deprecated
+  public void encode(T value, OutputStream outStream, Coder.Context context)
+      throws CoderException, IOException {
+    if (context == Coder.Context.NESTED) {
+      encode(value, outStream);
+    } else {
+      encodeOuter(value, outStream);
+    }
+  }
+
+  @Override
+  public T decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Coder.Context.NESTED);
+  }
+
+  @Deprecated
+  @Override
+  public T decodeOuter(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Coder.Context.OUTER);
+  }
+
+  @Deprecated
+  public T decode(InputStream inStream, Coder.Context context)
+      throws CoderException, IOException {
+    if (context == Coder.Context.NESTED) {
+      return decode(inStream);
+    } else {
+      return decodeOuter(inStream);
+    }
+  }
+
   /**
    * {@inheritDoc}.
    *

From 0afb140daaeaabded6a9873d4ab5710a400b2125 Mon Sep 17 00:00:00 2001
From: Pablo <pabloem@google.com>
Date: Wed, 3 May 2017 15:50:15 -0700
Subject: [PATCH 083/387] Metrics are queriable from Python SDK

---
 .../examples/cookbook/datastore_wordcount.py  |  13 +-
 sdks/python/apache_beam/examples/wordcount.py |   7 +-
 sdks/python/apache_beam/metrics/cells.py      |  16 +-
 sdks/python/apache_beam/metrics/execution.py  |   2 +-
 .../runners/dataflow/dataflow_metrics.py      | 114 +++++++---
 .../runners/dataflow/dataflow_metrics_test.py | 197 ++++++++++++------
 6 files changed, 253 insertions(+), 96 deletions(-)

diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
index 6e7ad76ec422..d694cdace600 100644
--- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
+++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
@@ -88,6 +88,7 @@ def __init__(self):
     self.empty_line_counter = Metrics.counter('main', 'empty_lines')
     self.word_length_counter = Metrics.counter('main', 'word_lengths')
     self.word_counter = Metrics.counter('main', 'total_words')
+    self.word_lengths_dist = Metrics.distribution('main', 'word_len_dist')
 
   def process(self, element):
     """Returns an iterator over words in contents of Cloud Datastore entity.
@@ -107,6 +108,7 @@ def process(self, element):
     words = re.findall(r'[A-Za-z\']+', text_line)
     for w in words:
       self.word_length_counter.inc(len(w))
+      self.word_len_dis.update(len(w))
       self.word_counter.inc()
     return words
 
@@ -254,7 +256,16 @@ def run(argv=None):
   if query_result['counters']:
     empty_lines_counter = query_result['counters'][0]
     logging.info('number of empty lines: %d', empty_lines_counter.committed)
-  # TODO(pabloem)(BEAM-1366): Add querying of MEAN metrics.
+  else:
+    logging.warn('unable to retrieve counter metrics from runner')
+
+  word_lengths_filter = MetricsFilter().with_name('word_len_dist')
+  query_result = result.metrics().query(word_lengths_filter)
+  if query_result['distributions']:
+    word_lengths_dist = query_result['distributions'][0]
+    logging.info('average word length: %d', word_lengths_dist.committed.mean)
+  else:
+    logging.warn('unable to retrieve distribution metrics from runner')
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py
index e93fd2b2a33f..adcc4db1908e 100644
--- a/sdks/python/apache_beam/examples/wordcount.py
+++ b/sdks/python/apache_beam/examples/wordcount.py
@@ -117,7 +117,12 @@ def _add_argparse_args(cls, parser):
     if query_result['counters']:
       empty_lines_counter = query_result['counters'][0]
       logging.info('number of empty lines: %d', empty_lines_counter.committed)
-    # TODO(pabloem)(BEAM-1366): Add querying of MEAN metrics.
+
+    word_lengths_filter = MetricsFilter().with_name('word_len_dist')
+    query_result = result.metrics().query(word_lengths_filter)
+    if query_result['distributions']:
+      word_lengths_dist = query_result['distributions'][0]
+      logging.info('average word length: %d', word_lengths_dist.committed.mean)
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/metrics/cells.py b/sdks/python/apache_beam/metrics/cells.py
index c4219493640f..41d24beda6dc 100644
--- a/sdks/python/apache_beam/metrics/cells.py
+++ b/sdks/python/apache_beam/metrics/cells.py
@@ -193,6 +193,13 @@ def __init__(self, data):
   def __eq__(self, other):
     return self.data == other.data
 
+  def __repr__(self):
+    return '<DistributionResult(sum={}, count={}, min={}, max={})>'.format(
+        self.sum,
+        self.count,
+        self.min,
+        self.max)
+
   @property
   def max(self):
     return self.data.max
@@ -244,10 +251,11 @@ def __neq__(self, other):
     return not self.__eq__(other)
 
   def __repr__(self):
-    return '<DistributionData({}, {}, {}, {})>'.format(self.sum,
-                                                       self.count,
-                                                       self.min,
-                                                       self.max)
+    return '<DistributionData(sum={}, count={}, min={}, max={})>'.format(
+        self.sum,
+        self.count,
+        self.min,
+        self.max)
 
   def get_cumulative(self):
     return DistributionData(self.sum, self.count, self.min, self.max)
diff --git a/sdks/python/apache_beam/metrics/execution.py b/sdks/python/apache_beam/metrics/execution.py
index 887423bc4b04..dbd05330d1fd 100644
--- a/sdks/python/apache_beam/metrics/execution.py
+++ b/sdks/python/apache_beam/metrics/execution.py
@@ -96,7 +96,7 @@ def __eq__(self, other):
 
   def __str__(self):
     return 'MetricResult(key={}, committed={}, attempted={})'.format(
-        self.key, self.committed, self.attempted)
+        self.key, str(self.committed), str(self.attempted))
 
 
 class _MetricsEnvironment(object):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py
index fe0122f8af09..24916fd49278 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py
@@ -22,14 +22,31 @@
 """
 
 from collections import defaultdict
-from warnings import warn
+import numbers
 
+from apache_beam.metrics.cells import DistributionData
+from apache_beam.metrics.cells import DistributionResult
 from apache_beam.metrics.execution import MetricKey
 from apache_beam.metrics.execution import MetricResult
 from apache_beam.metrics.metric import MetricResults
 from apache_beam.metrics.metricbase import MetricName
 
 
+def _get_match(proto, filter_fn):
+  """Finds and returns the first element that matches a query.
+
+  If no element matches the query, it throws ValueError.
+  If more than one element matches the query, it returns only the first.
+  """
+  query = [elm for elm in proto if filter_fn(elm)]
+  if len(query) == 0:
+    raise ValueError('Could not find element')
+  elif len(query) > 1:
+    raise ValueError('Too many matches')
+
+  return query[0]
+
+
 class DataflowMetrics(MetricResults):
   """Implementation of MetricResults class for the Dataflow runner."""
 
@@ -51,18 +68,25 @@ def __init__(self, dataflow_client=None, job_result=None, job_graph=None):
     self._cached_metrics = None
     self._job_graph = job_graph
 
+  @staticmethod
+  def _is_counter(metric_result):
+    return isinstance(metric_result.attempted, numbers.Number)
+
+  @staticmethod
+  def _is_distribution(metric_result):
+    return isinstance(metric_result.attempted, DistributionResult)
+
   def _translate_step_name(self, internal_name):
     """Translate between internal step names (e.g. "s1") and user step names."""
     if not self._job_graph:
       raise ValueError('Could not translate the internal step name.')
 
     try:
-      [step] = [step
-                for step in self._job_graph.proto.steps
-                if step.name == internal_name]
-      [user_step_name] = [prop.value.string_value
-                          for prop in step.properties.additionalProperties
-                          if prop.key == 'user_name']
+      step = _get_match(self._job_graph.proto.steps,
+                        lambda x: x.name == internal_name)
+      user_step_name = _get_match(
+          step.properties.additionalProperties,
+          lambda x: x.key == 'user_name').value.string_value
     except ValueError:
       raise ValueError('Could not translate the internal step name.')
     return user_step_name
@@ -77,19 +101,15 @@ def _get_metric_key(self, metric):
       #   step name (only happens for unstructured-named metrics).
       # 2. Unable to unpack [step] or [namespace]; which should only happen
       #   for unstructured names.
-      [step] = [prop.value
-                for prop in metric.name.context.additionalProperties
-                if prop.key == 'step']
+      step = _get_match(metric.name.context.additionalProperties,
+                        lambda x: x.key == 'step').value
       step = self._translate_step_name(step)
-      [namespace] = [prop.value
-                     for prop in metric.name.context.additionalProperties
-                     if prop.key == 'namespace']
+      namespace = _get_match(metric.name.context.additionalProperties,
+                             lambda x: x.key == 'namespace').value
       name = metric.name.name
     except ValueError:
-      # An unstructured metric name is "step/namespace/name", but step names
-      # can (and often do) contain slashes. Must only split on the right-most
-      # two slashes, to preserve the full step name.
-      [step, namespace, name] = metric.name.name.rsplit('/', 2)
+      return None
+
     return MetricKey(step, MetricName(namespace, name))
 
   def _populate_metric_results(self, response):
@@ -101,14 +121,17 @@ def _populate_metric_results(self, response):
     # Get the tentative/committed versions of every metric together.
     metrics_by_name = defaultdict(lambda: {})
     for metric in user_metrics:
-      if (metric.name.name.endswith('(DIST)') or
-          metric.name.name.endswith('[MIN]') or
+      if (metric.name.name.endswith('[MIN]') or
           metric.name.name.endswith('[MAX]') or
           metric.name.name.endswith('[MEAN]') or
           metric.name.name.endswith('[COUNT]')):
-        warn('Distribution metrics will be ignored in the MetricsResult.query'
-             'method. You can see them in the Dataflow User Interface.')
-        # Distributions are not yet fully supported in this runner
+        # The Dataflow Service presents distribution metrics in two ways:
+        # One way is as a single distribution object with all its fields, and
+        # another way is as four different scalar metrics labeled as [MIN],
+        # [MAX], [COUNT], [MEAN].
+        # TODO(pabloem) remove these when distributions are not being broken up
+        #  in the service.
+        # The second way is only useful for the UI, and should be ignored.
         continue
       is_tentative = [prop
                       for prop in metric.name.context.additionalProperties
@@ -116,22 +139,49 @@ def _populate_metric_results(self, response):
       tentative_or_committed = 'tentative' if is_tentative else 'committed'
 
       metric_key = self._get_metric_key(metric)
+      if metric_key is None:
+        continue
       metrics_by_name[metric_key][tentative_or_committed] = metric
 
     # Now we create the MetricResult elements.
     result = []
     for metric_key, metric in metrics_by_name.iteritems():
-      if (metric['tentative'].scalar is None or
-          metric['committed'].scalar is None):
+      attempted = self._get_metric_value(metric['tentative'])
+      committed = self._get_metric_value(metric['committed'])
+      if attempted is None or committed is None:
         continue
-      attempted = metric['tentative'].scalar.integer_value
-      committed = metric['committed'].scalar.integer_value
       result.append(MetricResult(metric_key,
                                  attempted=attempted,
                                  committed=committed))
 
     return result
 
+  def _get_metric_value(self, metric):
+    """Get a metric result object from a MetricUpdate from Dataflow API."""
+    if metric is None:
+      return None
+
+    if metric.scalar is not None:
+      return metric.scalar.integer_value
+    elif metric.distribution is not None:
+      dist_count = _get_match(metric.distribution.object_value.properties,
+                              lambda x: x.key == 'count').value.integer_value
+      dist_min = _get_match(metric.distribution.object_value.properties,
+                            lambda x: x.key == 'min').value.integer_value
+      dist_max = _get_match(metric.distribution.object_value.properties,
+                            lambda x: x.key == 'max').value.integer_value
+      dist_mean = _get_match(metric.distribution.object_value.properties,
+                             lambda x: x.key == 'mean').value.integer_value
+      # Calculating dist_sum with a hack, as distribution sum is not yet
+      # available in the Dataflow API.
+      # TODO(pabloem) Switch to "sum" field once it's available in the API
+      dist_sum = dist_count * dist_mean
+      return DistributionResult(
+          DistributionData(
+              dist_sum, dist_count, dist_min, dist_max))
+    else:
+      return None
+
   def _get_metrics_from_dataflow(self):
     """Return cached metrics or query the dataflow service."""
     try:
@@ -152,7 +202,11 @@ def _get_metrics_from_dataflow(self):
 
   def query(self, filter=None):
     response = self._get_metrics_from_dataflow()
-    counters = self._populate_metric_results(response)
-    # TODO(pabloem): Populate distributions once they are available.
-    return {'counters': [c for c in counters if self.matches(filter, c.key)],
-            'distributions': []}
+    metric_results = self._populate_metric_results(response)
+    return {'counters': [elm for elm in metric_results
+                         if self.matches(filter, elm.key)
+                         and DataflowMetrics._is_counter(elm)],
+            'distributions': [elm for elm in metric_results
+                              if self.matches(filter, elm.key)
+                              and DataflowMetrics._is_distribution(elm)],
+            'gauges': []} # Gauges are not currently supported by dataflow
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py
index 51a186d5c042..dd3cbe1156a4 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py
@@ -23,6 +23,8 @@
 
 import mock
 
+from apache_beam.metrics.cells import DistributionData
+from apache_beam.metrics.cells import DistributionResult
 from apache_beam.metrics.execution import MetricKey
 from apache_beam.metrics.execution import MetricResult
 from apache_beam.metrics.metricbase import MetricName
@@ -43,7 +45,7 @@ def _wrap(self, value):
 
 class TestDataflowMetrics(unittest.TestCase):
 
-  STRUCTURED_COUNTER_LIST = {"metrics": [
+  ONLY_COUNTERS_LIST = {"metrics": [
       {"name": {"context":
                 {"additionalProperties": [
                     {"key": "namespace",
@@ -53,10 +55,11 @@ class TestDataflowMetrics(unittest.TestCase):
                     {"key": "tentative",
                      "value": "true"}]
                 },
-                "name": "word_lengths",
+                "name": "words",
                 "origin": "user"
                },
-       "scalar": {"integer_value": 109475},
+       "scalar": {"integer_value": 26185},
+       "distribution": None,
        "updateTime": "2017-03-22T18:47:06.402Z"
       },
       {"name": {"context":
@@ -66,76 +69,137 @@ class TestDataflowMetrics(unittest.TestCase):
                     {"key": "step",
                      "value": "s2"}]
                 },
-                "name": "word_lengths",
+                "name": "words",
                 "origin": "user"
                },
-       "scalar": {"integer_value": 109475},
+       "scalar": {"integer_value": 26181},
+       "distribution": None,
        "updateTime": "2017-03-22T18:47:06.402Z"
       },
-  ]}
-
-  BASIC_COUNTER_LIST = {"metrics": [
       {"name": {"context":
-                {"additionalProperties":[
-                    {"key": "original_name",
-                     "value": "user-split-split/__main__.WordExtractingDoFn/"
-                              "empty_lines_TentativeAggregateValue"},
-                    {"key": "step", "value": "split"}]},
-                "name": "split/__main__.WordExtractingDoFn/empty_lines",
-                "origin": "user"},
+                {"additionalProperties": [
+                    {"key": "namespace",
+                     "value": "__main__.WordExtractingDoFn"},
+                    {"key": "step",
+                     "value": "s2"},
+                    {"key": "tentative",
+                     "value": "true"}]
+                },
+                "name": "empty_lines",
+                "origin": "user"
+               },
        "scalar": {"integer_value": 1080},
-       "updateTime": "2017-02-23T01:13:36.659Z"},
+       "distribution": None,
+       "updateTime": "2017-03-22T18:47:06.402Z"
+      },
       {"name": {"context":
                 {"additionalProperties": [
-                    {"key": "original_name",
-                     "value": "user-split-split/__main__.WordExtractingDoFn/"
-                              "empty_lines_TentativeAggregateValue"},
-                    {"key": "step", "value": "split"},
-                    {"key": "tentative", "value": "true"}]},
-                "name": "split/__main__.WordExtractingDoFn/empty_lines",
-                "origin": "user"},
+                    {"key": "namespace",
+                     "value": "__main__.WordExtractingDoFn"},
+                    {"key": "step",
+                     "value": "s2"}]
+                },
+                "name": "empty_lines",
+                "origin": "user"
+               },
        "scalar": {"integer_value": 1080},
-       "updateTime": "2017-02-23T01:13:36.659Z"},
+       "distribution": None,
+       "updateTime": "2017-03-22T18:47:06.402Z"
+      },
+  ]}
+  STRUCTURED_COUNTER_LIST = {"metrics": [
       {"name": {"context":
                 {"additionalProperties": [
-                    {"key": "original_name",
-                     "value": "user-split-split/__main__.WordExtractingDoFn/"
-                              "words_TentativeAggregateValue"},
-                    {"key": "step", "value": "split"}]},
-                "name": "longstepname/split/__main__.WordExtractingDoFn/words",
-                "origin": "user"},
-       "scalar": {"integer_value": 26181},
-       "updateTime": "2017-02-23T01:13:36.659Z"},
+                    {"key": "namespace",
+                     "value": "__main__.WordExtractingDoFn"},
+                    {"key": "step",
+                     "value": "s2"},
+                    {"key": "tentative",
+                     "value": "true"}]
+                },
+                "name": "word_lengths",
+                "origin": "user"
+               },
+       "scalar": {"integer_value": 109475},
+       "distribution": None,
+       "updateTime": "2017-03-22T18:47:06.402Z"
+      },
       {"name": {"context":
                 {"additionalProperties": [
-                    {"key": "original_name",
-                     "value": "user-split-longstepname/split/"
-                              "__main__.WordExtractingDoFn/"
-                              "words_TentativeAggregateValue"},
-                    {"key": "step", "value": "split"},
-                    {"key": "tentative", "value": "true"}]},
-                "name": "longstepname/split/__main__.WordExtractingDoFn/words",
-                "origin": "user"},
-       "scalar": {"integer_value": 26185},
-       "updateTime": "2017-02-23T01:13:36.659Z"},
+                    {"key": "namespace",
+                     "value": "__main__.WordExtractingDoFn"},
+                    {"key": "step",
+                     "value": "s2"}]
+                },
+                "name": "word_lengths",
+                "origin": "user"
+               },
+       "scalar": {"integer_value": 109475},
+       "distribution": None,
+       "updateTime": "2017-03-22T18:47:06.402Z"
+      },
       {"name": {"context":
                 {"additionalProperties": [
-                    {"key": "original_name",
-                     "value": "user-split-split/__main__.WordExtractingDoFn/"
-                              "secretdistribution(DIST)"},
-                    {"key": "step", "value": "split"},
-                    {"key": "tentative", "value": "true"}]},
-                "name":
-                "split/__main__.WordExtractingDoFn/secretdistribution(DIST)",
-                "origin": "user"},
-       "scalar": {"integer_value": 15},
-       "updateTime": "2017-02-23T01:13:36.659Z"}
+                    {"key": "namespace",
+                     "value": "__main__.WordExtractingDoFn"},
+                    {"key": "step",
+                     "value": "s2"},
+                    {"key": "tentative",
+                     "value": "true"}]
+                },
+                "name": "word_length_dist",
+                "origin": "user"
+               },
+       "scalar": None,
+       "distribution": {
+           "object_value": {
+               "properties": [
+                   {"key": "min", "value":
+                    {"integer_value": 2}},
+                   {"key": "max", "value":
+                    {"integer_value": 16}},
+                   {"key": "count", "value":
+                    {"integer_value": 2}},
+                   {"key": "mean", "value":
+                    {"integer_value": 9}},
+                   {"key": "sum", "value":
+                    {"integer_value": 18}},]
+           }
+       },
+       "updateTime": "2017-03-22T18:47:06.402Z"
+      },
+      {"name": {"context":
+                {"additionalProperties": [
+                    {"key": "namespace",
+                     "value": "__main__.WordExtractingDoFn"},
+                    {"key": "step",
+                     "value": "s2"}]
+                },
+                "name": "word_length_dist",
+                "origin": "user"
+               },
+       "scalar": None,
+       "distribution": {
+           "object_value": {
+               "properties": [
+                   {"key": "min", "value":
+                    {"integer_value": 2}},
+                   {"key": "max", "value":
+                    {"integer_value": 16}},
+                   {"key": "count", "value":
+                    {"integer_value": 2}},
+                   {"key": "mean", "value":
+                    {"integer_value": 9}},
+                   {"key": "sum", "value":
+                    {"integer_value": 18}},
+               ]
+           }
+       },
+       "updateTime": "2017-03-22T18:47:06.402Z"
+      },
   ]}
 
   def setup_mock_client_result(self, counter_list=None):
-    if counter_list is None:
-      counter_list = self.BASIC_COUNTER_LIST
-
     mock_client = mock.Mock()
     mock_query_result = DictToObject(counter_list)
     mock_client.get_job_metrics.return_value = mock_query_result
@@ -145,7 +209,8 @@ def setup_mock_client_result(self, counter_list=None):
     return mock_client, mock_job_result
 
   def test_cache_functions(self):
-    mock_client, mock_job_result = self.setup_mock_client_result()
+    mock_client, mock_job_result = self.setup_mock_client_result(
+        self.STRUCTURED_COUNTER_LIST)
     dm = dataflow_metrics.DataflowMetrics(mock_client, mock_job_result)
 
     # At first creation, we should always query dataflow.
@@ -160,7 +225,7 @@ def test_cache_functions(self):
     dm.query()
     self.assertTrue(dm._cached_metrics)
 
-  def test_query_structured_counters(self):
+  def test_query_structured_metrics(self):
     mock_client, mock_job_result = self.setup_mock_client_result(
         self.STRUCTURED_COUNTER_LIST)
     dm = dataflow_metrics.DataflowMetrics(mock_client, mock_job_result)
@@ -175,9 +240,23 @@ def test_query_structured_counters(self):
         ]
     self.assertEqual(query_result['counters'], expected_counters)
 
+    expected_distributions = [
+        MetricResult(
+            MetricKey('split',
+                      MetricName('__main__.WordExtractingDoFn',
+                                 'word_length_dist')),
+            DistributionResult(DistributionData(
+                18, 2, 2, 16)),
+            DistributionResult(DistributionData(
+                18, 2, 2, 16))),
+        ]
+    self.assertEqual(query_result['distributions'], expected_distributions)
+
   def test_query_counters(self):
-    mock_client, mock_job_result = self.setup_mock_client_result()
+    mock_client, mock_job_result = self.setup_mock_client_result(
+        self.ONLY_COUNTERS_LIST)
     dm = dataflow_metrics.DataflowMetrics(mock_client, mock_job_result)
+    dm._translate_step_name = types.MethodType(lambda self, x: 'split', dm)
     query_result = dm.query()
     expected_counters = [
         MetricResult(
@@ -185,7 +264,7 @@ def test_query_counters(self):
                       MetricName('__main__.WordExtractingDoFn', 'empty_lines')),
             1080, 1080),
         MetricResult(
-            MetricKey('longstepname/split',
+            MetricKey('split',
                       MetricName('__main__.WordExtractingDoFn', 'words')),
             26181, 26185),
         ]

From 2e379232a7bc92f9af286646e3e7aa7c84218fd9 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 09:45:50 -0700
Subject: [PATCH 084/387] Archetypes: fix build order

---
 .../maven-archetypes/examples-java8/pom.xml   | 83 +------------------
 sdks/java/maven-archetypes/examples/pom.xml   | 65 +--------------
 sdks/java/maven-archetypes/starter/pom.xml    |  9 +-
 3 files changed, 14 insertions(+), 143 deletions(-)

diff --git a/sdks/java/maven-archetypes/examples-java8/pom.xml b/sdks/java/maven-archetypes/examples-java8/pom.xml
index 01f7270bf810..49b2d1ba8a54 100644
--- a/sdks/java/maven-archetypes/examples-java8/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/pom.xml
@@ -94,88 +94,13 @@
     </plugins>
   </build>
 
-  <!--
-    These are the dependencies of the generated code. This enables
-    the include-what-you-use analysis to work on the archetype.
-
-    In alphabetical order by groupId, then artifactId.
-  -->
   <dependencies>
-    <dependency>
-      <groupId>com.google.api-client</groupId>
-      <artifactId>google-api-client</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.apis</groupId>
-      <artifactId>google-api-services-bigquery</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.apis</groupId>
-      <artifactId>google-api-services-pubsub</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.oauth-client</groupId>
-      <artifactId>google-oauth-client</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>joda-time</groupId>
-      <artifactId>joda-time</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-java-core</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-runners-direct-java</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
+    <!-- Ensure that the examples-java8 module is up to date before compiling
+         and testing this archetype so that the tested code receives the
+         correct dependencies. -->
     <dependency>
       <groupId>org.apache.beam</groupId>
-      <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
+      <artifactId>beam-examples-java8</artifactId>
       <scope>runtime</scope>
     </dependency>
   </dependencies>
diff --git a/sdks/java/maven-archetypes/examples/pom.xml b/sdks/java/maven-archetypes/examples/pom.xml
index 3483b2a60d27..54cac44cd6ba 100644
--- a/sdks/java/maven-archetypes/examples/pom.xml
+++ b/sdks/java/maven-archetypes/examples/pom.xml
@@ -95,69 +95,12 @@
   </build>
 
   <dependencies>
+    <!-- Ensure that the examples-java module is up to date before compiling
+         and testing this archetype so that the tested code receives the
+         correct dependencies. -->
     <dependency>
       <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-java-core</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-runners-direct-java</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>joda-time</groupId>
-      <artifactId>joda-time</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.apis</groupId>
-      <artifactId>google-api-services-bigquery</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.apis</groupId>
-      <artifactId>google-api-services-pubsub</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.api-client</groupId>
-      <artifactId>google-api-client</artifactId>
+      <artifactId>beam-examples-java</artifactId>
       <scope>runtime</scope>
     </dependency>
   </dependencies>
diff --git a/sdks/java/maven-archetypes/starter/pom.xml b/sdks/java/maven-archetypes/starter/pom.xml
index dcda9e00fe42..0ef81b3855fc 100644
--- a/sdks/java/maven-archetypes/starter/pom.xml
+++ b/sdks/java/maven-archetypes/starter/pom.xml
@@ -81,15 +81,18 @@
   </build>
 
   <dependencies>
+    <!-- List here all Apache Beam modules that the archetype's source pom.xml
+         depends on. This ensures that those modules are up-to-date before the
+         starter archetype is compiled and tested. -->
     <dependency>
       <groupId>org.apache.beam</groupId>
-      <artifactId>beam-sdks-java-core</artifactId>
+      <artifactId>beam-runners-direct-java</artifactId>
       <scope>runtime</scope>
     </dependency>
 
     <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-core</artifactId>
       <scope>runtime</scope>
     </dependency>
   </dependencies>

From e6ee29917813f12b0de086af5adaa8d6c05c31ba Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 11:39:51 -0700
Subject: [PATCH 085/387] Examples: allow java8 bytecode since the examples are
 java8

---
 .../maven-archetypes/examples-java8/pom.xml   | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/sdks/java/maven-archetypes/examples-java8/pom.xml b/sdks/java/maven-archetypes/examples-java8/pom.xml
index 49b2d1ba8a54..79b4d462267d 100644
--- a/sdks/java/maven-archetypes/examples-java8/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/pom.xml
@@ -91,6 +91,29 @@
           </execution>
         </executions>
       </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>enforce</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <enforceBytecodeVersion>
+                  <maxJdkVersion>1.8</maxJdkVersion>
+                </enforceBytecodeVersion>
+                <requireJavaVersion>
+                  <version>[1.8,)</version>
+                </requireJavaVersion>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

From cbe182eee203c92073a25bbcd39f86f8440a5fc2 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 5 May 2017 11:00:58 -0700
Subject: [PATCH 086/387] [BEAM-1585] Filesystems should be picked using the
 provided scheme

---
 sdks/python/apache_beam/io/fileio_test.py     |  3 +-
 sdks/python/apache_beam/io/filesystem.py      | 16 +++++++++
 sdks/python/apache_beam/io/filesystems.py     | 36 +++++++++++++++++--
 .../python/apache_beam/io/filesystems_test.py | 15 +++++++-
 .../python/apache_beam/io/filesystems_util.py | 36 -------------------
 .../apache_beam/io/gcp/gcsfilesystem.py       |  6 ++++
 .../apache_beam/io/gcp/gcsfilesystem_test.py  |  5 +++
 sdks/python/apache_beam/io/localfilesystem.py |  5 +++
 .../apache_beam/io/localfilesystem_test.py    |  4 +++
 9 files changed, 84 insertions(+), 42 deletions(-)
 delete mode 100644 sdks/python/apache_beam/io/filesystems_util.py

diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 13778d53f45f..3f7211ede359 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -31,7 +31,6 @@
 import apache_beam as beam
 from apache_beam import coders
 from apache_beam.io import fileio
-from apache_beam.io.filesystem import BeamIOError
 from apache_beam.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
@@ -216,7 +215,7 @@ def test_temp_dir_gcs(self):
       self.run_temp_dir_check(
           'gs://aaa/bbb', 'gs://aaa/bbb/', 'gs://aaa', 'gs://aaa/', 'gs://',
           '/')
-    except BeamIOError:
+    except ValueError:
       logging.debug('Ignoring test since GCP module is not installed')
 
   @mock.patch('apache_beam.io.localfilesystem.os')
diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py
index ff8af03af64b..3d35f3e49ce4 100644
--- a/sdks/python/apache_beam/io/filesystem.py
+++ b/sdks/python/apache_beam/io/filesystem.py
@@ -425,6 +425,22 @@ def _get_compression_type(path, compression_type):
                       'was %s' % type(compression_type))
     return compression_type
 
+  @classmethod
+  def get_all_subclasses(cls):
+    """Get all the subclasses of the FileSystem class
+    """
+    all_subclasses = []
+    for subclass in cls.__subclasses__():
+      all_subclasses.append(subclass)
+      all_subclasses.extend(subclass.get_all_subclasses())
+    return all_subclasses
+
+  @classmethod
+  def scheme(cls):
+    """URI scheme for the FileSystem
+    """
+    raise NotImplementedError
+
   @abc.abstractmethod
   def join(self, basepath, *paths):
     """Join two or more pathname components for the filesystem
diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py
index 225a85757588..29f06442a9a1 100644
--- a/sdks/python/apache_beam/io/filesystems.py
+++ b/sdks/python/apache_beam/io/filesystems.py
@@ -17,24 +17,54 @@
 
 """FileSystems interface class for accessing the correct filesystem"""
 
+import re
+
 from apache_beam.io.filesystem import BeamIOError
 from apache_beam.io.filesystem import CompressionTypes
-from apache_beam.io.filesystems_util import get_filesystem
+from apache_beam.io.filesystem import FileSystem
+
+# All filesystem implements should be added here
+# pylint: disable=wrong-import-position, unused-import
+from apache_beam.io.localfilesystem import LocalFileSystem
+
+try:
+  from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
+except ImportError:
+  pass
+# pylint: enable=wrong-import-position, unused-import
 
 
 class FileSystems(object):
   """A class that defines the functions that can be performed on a filesystem.
   All methods are static and access the underlying registered filesystems.
   """
+  URI_SCHEMA_PATTERN = re.compile('(?P<scheme>[a-zA-Z][-a-zA-Z0-9+.]*)://.*')
+
+  @staticmethod
+  def get_scheme(path):
+    match_result = FileSystems.URI_SCHEMA_PATTERN.match(path.strip())
+    if match_result is None:
+      return None
+    return match_result.groupdict()['scheme']
 
   @staticmethod
   def get_filesystem(path):
     """Get the correct filesystem for the specified path
     """
     try:
-      return get_filesystem(path)
+      path_scheme = FileSystems.get_scheme(path)
+      systems = [fs for fs in FileSystem.get_all_subclasses()
+                 if fs.scheme() == path_scheme]
+      if len(systems) == 0:
+        raise ValueError('Unable to get the Filesystem for path %s' % path)
+      elif len(systems) == 1:
+        return systems[0]()
+      else:
+        raise ValueError('Found more than one filesystem for path %s' % path)
+    except ValueError:
+      raise
     except Exception as e:
-      raise BeamIOError('Enable to get the Filesystem', {path: e})
+      raise BeamIOError('Unable to get the Filesystem', {path: e})
 
   @staticmethod
   def join(basepath, *paths):
diff --git a/sdks/python/apache_beam/io/filesystems_test.py b/sdks/python/apache_beam/io/filesystems_test.py
index 91655860bb42..9a6f013ff479 100644
--- a/sdks/python/apache_beam/io/filesystems_test.py
+++ b/sdks/python/apache_beam/io/filesystems_test.py
@@ -48,6 +48,19 @@ def setUp(self):
   def tearDown(self):
     shutil.rmtree(self.tmpdir)
 
+  def test_get_scheme(self):
+    self.assertIsNone(FileSystems.get_scheme('/abc/cdf'))
+    self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf'))  # pylint: disable=anomalous-backslash-in-string
+    self.assertEqual(FileSystems.get_scheme('gs://abc/cdf'), 'gs')
+
+  def test_get_filesystem(self):
+    self.assertTrue(isinstance(FileSystems.get_filesystem('/tmp'),
+                               localfilesystem.LocalFileSystem))
+    self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'),  # pylint: disable=anomalous-backslash-in-string
+                               localfilesystem.LocalFileSystem))
+    with self.assertRaises(ValueError):
+      FileSystems.get_filesystem('error://abc/def')
+
   @mock.patch('apache_beam.io.localfilesystem.os')
   def test_unix_path_join(self, *unused_mocks):
     # Test joining of Unix paths.
@@ -110,7 +123,7 @@ def test_match_file_exception(self):
     with self.assertRaises(BeamIOError) as error:
       FileSystems.match([None])
     self.assertTrue(
-        error.exception.message.startswith('Enable to get the Filesystem'))
+        error.exception.message.startswith('Unable to get the Filesystem'))
     self.assertEqual(error.exception.exception_details.keys(), [None])
 
   def test_match_directory(self):
diff --git a/sdks/python/apache_beam/io/filesystems_util.py b/sdks/python/apache_beam/io/filesystems_util.py
deleted file mode 100644
index 6d21298a6779..000000000000
--- a/sdks/python/apache_beam/io/filesystems_util.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-"""Utility functions for getting the correct file systems for a file name"""
-
-from apache_beam.io.localfilesystem import LocalFileSystem
-
-
-# TODO(BEAM-1585): Add a mechanism to add user implemented file systems
-def get_filesystem(path):
-  """Function that returns the FileSystem class to use based on the path
-  provided in the input.
-  """
-  if path.startswith('gs://'):
-    try:
-      from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
-    except ImportError:
-      raise ImportError(
-          'Google Cloud Platform IO not available, '
-          'please install apache_beam[gcp]')
-    return GCSFileSystem()
-  else:
-    return LocalFileSystem()
diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
index ba00f5080929..dc71fcec6459 100644
--- a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
+++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
@@ -34,6 +34,12 @@ class GCSFileSystem(FileSystem):
   CHUNK_SIZE = gcsio.MAX_BATCH_OPERATION_SIZE  # Chuck size in batch operations
   GCS_PREFIX = 'gs://'
 
+  @classmethod
+  def scheme(cls):
+    """URI scheme for the FileSystem
+    """
+    return 'gs'
+
   def join(self, basepath, *paths):
     """Join two or more pathname components for the filesystem
 
diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
index 5fb9a6289686..923fc7dd4b68 100644
--- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
@@ -36,6 +36,11 @@
 @unittest.skipIf(gcsfilesystem is None, 'GCP dependencies are not installed')
 class GCSFileSystemTest(unittest.TestCase):
 
+  def test_scheme(self):
+    file_system = gcsfilesystem.GCSFileSystem()
+    self.assertEqual(file_system.scheme(), 'gs')
+    self.assertEqual(gcsfilesystem.GCSFileSystem.scheme(), 'gs')
+
   def test_join(self):
     file_system = gcsfilesystem.GCSFileSystem()
     self.assertEqual('gs://bucket/path/to/file',
diff --git a/sdks/python/apache_beam/io/localfilesystem.py b/sdks/python/apache_beam/io/localfilesystem.py
index bc7c576cf749..c6707041b8e5 100644
--- a/sdks/python/apache_beam/io/localfilesystem.py
+++ b/sdks/python/apache_beam/io/localfilesystem.py
@@ -33,6 +33,11 @@
 class LocalFileSystem(FileSystem):
   """A Local ``FileSystem`` implementation for accessing files on disk.
   """
+  @classmethod
+  def scheme(cls):
+    """URI scheme for the FileSystem
+    """
+    return None
 
   def join(self, basepath, *paths):
     """Join two or more pathname components for the filesystem
diff --git a/sdks/python/apache_beam/io/localfilesystem_test.py b/sdks/python/apache_beam/io/localfilesystem_test.py
index 986dcaf05bda..04cf5b7c6f4d 100644
--- a/sdks/python/apache_beam/io/localfilesystem_test.py
+++ b/sdks/python/apache_beam/io/localfilesystem_test.py
@@ -61,6 +61,10 @@ def setUp(self):
   def tearDown(self):
     shutil.rmtree(self.tmpdir)
 
+  def test_scheme(self):
+    self.assertIsNone(self.fs.scheme())
+    self.assertIsNone(localfilesystem.LocalFileSystem.scheme())
+
   @mock.patch('apache_beam.io.localfilesystem.os')
   def test_unix_path_join(self, *unused_mocks):
     # Test joining of Unix paths.

From ae70e50c7ed639e5607dbfc9e9534b9a5eff10bc Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Wed, 3 May 2017 17:15:52 -0700
Subject: [PATCH 087/387] [BEAM-2165] Update Spark to support
 serializing/deserializing custom user types configured via Jackson modules

---
 .../translation/SparkRuntimeContext.java      |  17 ++-
 .../translation/SparkRuntimeContextTest.java  | 122 ++++++++++++++++++
 2 files changed, 137 insertions(+), 2 deletions(-)
 create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index 3db1ab51ddbb..970a35626f07 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -19,6 +19,7 @@
 package org.apache.beam.runners.spark.translation;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.io.Serializable;
@@ -26,6 +27,8 @@
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.common.ReflectHelpers;
 
 /**
  * The SparkRuntimeContext allows us to define useful features on the client side before our
@@ -39,9 +42,19 @@ public class SparkRuntimeContext implements Serializable {
     this.serializedPipelineOptions = serializePipelineOptions(options);
   }
 
+  /**
+   * Use an {@link ObjectMapper} configured with any {@link Module}s in the class path allowing
+   * for user specified configuration injection into the ObjectMapper. This supports user custom
+   * types on {@link PipelineOptions}.
+   */
+  private static ObjectMapper createMapper() {
+    return new ObjectMapper().registerModules(
+        ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
+  }
+
   private String serializePipelineOptions(PipelineOptions pipelineOptions) {
     try {
-      return new ObjectMapper().writeValueAsString(pipelineOptions);
+      return createMapper().writeValueAsString(pipelineOptions);
     } catch (JsonProcessingException e) {
       throw new IllegalStateException("Failed to serialize the pipeline options.", e);
     }
@@ -49,7 +62,7 @@ private String serializePipelineOptions(PipelineOptions pipelineOptions) {
 
   private static PipelineOptions deserializePipelineOptions(String serializedPipelineOptions) {
     try {
-      return new ObjectMapper().readValue(serializedPipelineOptions, PipelineOptions.class);
+      return createMapper().readValue(serializedPipelineOptions, PipelineOptions.class);
     } catch (IOException e) {
       throw new IllegalStateException("Failed to deserialize the pipeline options.", e);
     }
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
new file mode 100644
index 000000000000..372d46f26765
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.translation;
+
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.auto.service.AutoService;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.CrashingRunner;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link SparkRuntimeContext}.
+ */
+@RunWith(JUnit4.class)
+public class SparkRuntimeContextTest {
+  /** PipelineOptions used to test auto registration of Jackson modules. */
+  public interface JacksonIncompatibleOptions extends PipelineOptions {
+    JacksonIncompatible getJacksonIncompatible();
+    void setJacksonIncompatible(JacksonIncompatible value);
+  }
+
+  /** A Jackson {@link Module} to test auto-registration of modules. */
+  @AutoService(Module.class)
+  public static class RegisteredTestModule extends SimpleModule {
+    public RegisteredTestModule() {
+      super("RegisteredTestModule");
+      setMixInAnnotation(JacksonIncompatible.class, JacksonIncompatibleMixin.class);
+    }
+  }
+
+  /** A class which Jackson does not know how to serialize/deserialize. */
+  public static class JacksonIncompatible {
+    private final String value;
+    public JacksonIncompatible(String value) {
+      this.value = value;
+    }
+  }
+
+  /** A Jackson mixin used to add annotations to other classes. */
+  @JsonDeserialize(using = JacksonIncompatibleDeserializer.class)
+  @JsonSerialize(using = JacksonIncompatibleSerializer.class)
+  public static final class JacksonIncompatibleMixin {}
+
+  /** A Jackson deserializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleDeserializer extends
+      JsonDeserializer<JacksonIncompatible> {
+
+    @Override
+    public JacksonIncompatible deserialize(JsonParser jsonParser,
+        DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+      return new JacksonIncompatible(jsonParser.readValueAs(String.class));
+    }
+  }
+
+  /** A Jackson serializer for {@link JacksonIncompatible}. */
+  public static class JacksonIncompatibleSerializer extends JsonSerializer<JacksonIncompatible> {
+
+    @Override
+    public void serialize(JacksonIncompatible jacksonIncompatible, JsonGenerator jsonGenerator,
+        SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
+      jsonGenerator.writeString(jacksonIncompatible.value);
+    }
+  }
+
+  @Test
+  public void testSerializingPipelineOptionsWithCustomUserType() throws Exception {
+    PipelineOptions options = PipelineOptionsFactory.fromArgs("--jacksonIncompatible=\"testValue\"")
+        .as(JacksonIncompatibleOptions.class);
+    options.setRunner(CrashingRunner.class);
+    Pipeline p = Pipeline.create(options);
+    SparkRuntimeContext context = new SparkRuntimeContext(p);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    try (ObjectOutputStream outputStream = new ObjectOutputStream(baos)) {
+      outputStream.writeObject(context);
+    }
+    try (ObjectInputStream inputStream =
+        new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()))) {
+      SparkRuntimeContext copy = (SparkRuntimeContext) inputStream.readObject();
+      assertEquals("testValue",
+          copy.getPipelineOptions().as(JacksonIncompatibleOptions.class)
+              .getJacksonIncompatible().value);
+    }
+  }
+}

From b709f9ad33ef580a55228d6f32d93308f336a6cc Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 14:41:23 -0700
Subject: [PATCH 088/387] Fix merge bug in SparkRuntimeContext

---
 .../beam/runners/spark/translation/SparkRuntimeContext.java      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index 970a35626f07..82e8b440ab8d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -27,7 +27,6 @@
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
 
 /**

From f8e2cf89febeb2f86d2dc91c8d3fff5d43df3623 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Thu, 4 May 2017 19:55:47 -0700
Subject: [PATCH 089/387] [BEAM-2174] Update CoderRegistry to allow creating
 coders through CoderFactory for a wider range of types

---
 .../apache/beam/examples/complete/TfIdf.java  |   2 +-
 .../beam/examples/complete/TfIdfTest.java     |   2 +-
 .../beam/runners/core/SplittableParDo.java    |   4 +-
 ...opyOnAccessInMemoryStateInternalsTest.java |   2 +-
 .../beam/runners/direct/DirectRunnerTest.java |  10 -
 .../apache/beam/sdk/coders/AtomicCoder.java   |  14 -
 .../org/apache/beam/sdk/coders/AvroCoder.java |  42 +-
 .../beam/sdk/coders/ByteArrayCoder.java       |   8 -
 .../coders/CannotProvideCoderException.java   |   2 +-
 .../org/apache/beam/sdk/coders/Coder.java     |   4 +-
 .../beam/sdk/coders/CoderFactories.java       | 292 ---------
 .../apache/beam/sdk/coders/CoderFactory.java  |  44 --
 .../apache/beam/sdk/coders/CoderProvider.java |  19 +-
 ...strar.java => CoderProviderRegistrar.java} |  13 +-
 .../beam/sdk/coders/CoderProviders.java       | 240 ++++---
 .../apache/beam/sdk/coders/CoderRegistry.java | 618 ++++++------------
 .../beam/sdk/coders/CollectionCoder.java      |   9 -
 .../apache/beam/sdk/coders/CustomCoder.java   |   8 -
 .../apache/beam/sdk/coders/DefaultCoder.java  | 119 +++-
 .../apache/beam/sdk/coders/IterableCoder.java |   9 -
 .../beam/sdk/coders/IterableLikeCoder.java    |  12 -
 .../org/apache/beam/sdk/coders/KvCoder.java   |   7 -
 .../org/apache/beam/sdk/coders/ListCoder.java |   8 -
 .../org/apache/beam/sdk/coders/MapCoder.java  |  12 -
 .../beam/sdk/coders/SerializableCoder.java    |  57 +-
 .../org/apache/beam/sdk/coders/SetCoder.java  |   9 -
 .../apache/beam/sdk/coders/VarLongCoder.java  |   7 -
 .../beam/sdk/transforms/CombineFnBase.java    |   4 +-
 .../beam/sdk/transforms/CombineFns.java       |   4 +-
 .../apache/beam/sdk/transforms/Create.java    |  76 ++-
 .../beam/sdk/transforms/PTransform.java       |   4 +-
 .../org/apache/beam/sdk/transforms/ParDo.java |   8 +-
 .../apache/beam/sdk/transforms/WithKeys.java  |   4 +-
 .../transforms/windowing/GlobalWindow.java    |   6 -
 .../transforms/windowing/IntervalWindow.java  |   7 -
 .../apache/beam/sdk/values/PCollection.java   |   2 +-
 .../beam/sdk/values/TimestampedValue.java     |   4 -
 .../beam/sdk/coders/CoderFactoriesTest.java   | 100 ---
 .../beam/sdk/coders/CoderProvidersTest.java   |  82 ++-
 .../beam/sdk/coders/CoderRegistryTest.java    | 167 ++---
 .../beam/sdk/coders/DefaultCoderTest.java     |  65 +-
 .../beam/sdk/coders/IterableCoderTest.java    |  17 -
 .../apache/beam/sdk/coders/ListCoderTest.java |  17 -
 .../apache/beam/sdk/coders/MapCoderTest.java  |  20 -
 .../sdk/coders/SerializableCoderTest.java     |  11 +
 .../beam/sdk/transforms/CombineFnsTest.java   |  10 +-
 .../beam/sdk/transforms/CreateTest.java       |   6 +-
 .../sdk/transforms/FlatMapElementsTest.java   |   4 +-
 .../beam/sdk/transforms/GroupByKeyTest.java   |   4 +-
 .../beam/sdk/transforms/MapElementsTest.java  |   4 +-
 .../apache/beam/sdk/transforms/ParDoTest.java |  17 +-
 .../transforms/reflect/DoFnInvokersTest.java  |   4 +-
 .../beam/sdk/values/TypedPValueTest.java      |  28 +-
 .../sdk/extensions/protobuf/ProtoCoder.java   |  72 +-
 ...va => ProtobufCoderProviderRegistrar.java} |  24 +-
 .../extensions/protobuf/ProtoCoderTest.java   |   7 +-
 ...va => BigQueryCoderProviderRegistrar.java} |  25 +-
 .../io/gcp/bigquery/DynamicDestinations.java  |   2 +-
 ...java => PubsubCoderProviderRegistrar.java} |  24 +-
 ...> BigQueryCoderProviderRegistrarTest.java} |   8 +-
 sdks/java/io/hadoop-common/pom.xml            |   7 +
 .../beam/sdk/io/hadoop/WritableCoder.java     |  56 ++
 .../beam/sdk/io/hadoop/WritableCoderTest.java |  10 +
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java |   2 +-
 .../sdk/transforms/DistinctJava8Test.java     |   8 -
 .../beam/sdk/transforms/FilterJava8Test.java  |   2 +-
 .../sdk/transforms/PartitionJava8Test.java    |   2 +-
 .../sdk/transforms/WithKeysJava8Test.java     |   9 -
 68 files changed, 969 insertions(+), 1537 deletions(-)
 delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
 delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactory.java
 rename sdks/java/core/src/main/java/org/apache/beam/sdk/coders/{CoderRegistrar.java => CoderProviderRegistrar.java} (76%)
 delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderFactoriesTest.java
 rename sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/{ProtobufCoderRegistrar.java => ProtobufCoderProviderRegistrar.java} (58%)
 rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/{BigQueryCoderRegistrar.java => BigQueryCoderProviderRegistrar.java} (56%)
 rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/{PubsubCoderRegistrar.java => PubsubCoderProviderRegistrar.java} (56%)
 rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/{BigQueryCoderRegistrarTest.java => BigQueryCoderProviderRegistrarTest.java} (84%)

diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
index 6fd97554d5bd..7552b9477f57 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
@@ -409,7 +409,7 @@ public void processElement(ProcessContext c) {
   public static void main(String[] args) throws Exception {
     Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
     Pipeline pipeline = Pipeline.create(options);
-    pipeline.getCoderRegistry().registerCoder(URI.class, StringDelegateCoder.of(URI.class));
+    pipeline.getCoderRegistry().registerCoderForClass(URI.class, StringDelegateCoder.of(URI.class));
 
     pipeline
         .apply(new ReadDocuments(listInputDocuments(options)))
diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/TfIdfTest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/TfIdfTest.java
index d263643807a0..3681ff54b00d 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/complete/TfIdfTest.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/complete/TfIdfTest.java
@@ -48,7 +48,7 @@ public class TfIdfTest {
   @Category(ValidatesRunner.class)
   public void testTfIdf() throws Exception {
 
-    pipeline.getCoderRegistry().registerCoder(URI.class, StringDelegateCoder.of(URI.class));
+    pipeline.getCoderRegistry().registerCoderForClass(URI.class, StringDelegateCoder.of(URI.class));
 
     PCollection<KV<String, KV<URI, Double>>> wordToUriAndTfIdf = pipeline
         .apply(Create.of(
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
index a4d96392bf2a..6503fa20938d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
@@ -274,7 +274,7 @@ public <T> Coder<T> getDefaultOutputCoder(
             input,
         PCollection<T> output)
         throws CannotProvideCoderException {
-      // Similar logic to ParDo.MultiOutput.getDefaultOutputCoder.
+      // Similar logic to ParDo.MultiOutput.getOutputCoder.
       @SuppressWarnings("unchecked")
       KeyedWorkItemCoder<String, ElementAndRestriction<InputT, RestrictionT>> kwiCoder =
           (KeyedWorkItemCoder) input.getCoder();
@@ -284,7 +284,7 @@ public <T> Coder<T> getDefaultOutputCoder(
       return input
           .getPipeline()
           .getCoderRegistry()
-          .getDefaultCoder(output.getTypeDescriptor(), fn.getInputTypeDescriptor(), inputCoder);
+          .getCoder(output.getTypeDescriptor(), fn.getInputTypeDescriptor(), inputCoder);
     }
   }
 
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
index 46f26a1d4701..1e60ca33ef8f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java
@@ -229,7 +229,7 @@ public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCo
     CoderRegistry reg = pipeline.getCoderRegistry();
     StateTag<CombiningState<Long, long[], Long>> stateTag =
         StateTags.combiningValue("summer",
-            sumLongFn.getAccumulatorCoder(reg, reg.getDefaultCoder(Long.class)), sumLongFn);
+            sumLongFn.getAccumulatorCoder(reg, reg.getCoder(Long.class)), sumLongFn);
     GroupingState<Long, Long> underlyingValue = underlying.state(namespace, stateTag);
     assertThat(underlyingValue.read(), equalTo(0L));
 
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index 0fe958515465..85e55eb3a30c 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -577,14 +577,4 @@ public Coder<T> getDefaultOutputCoder() {
       return underlying.getDefaultOutputCoder();
     }
   }
-
-  @Test
-  public void fallbackCoderProviderAllowsInference() {
-    // See https://issues.apache.org/jira/browse/BEAM-1642
-    Pipeline p = getPipeline();
-    p.getCoderRegistry().setFallbackCoderProvider(
-        org.apache.beam.sdk.coders.AvroCoder.PROVIDER);
-    p.apply(Create.of(Arrays.asList(100, 200))).apply(Count.<Integer>globally());
-    p.run().waitUntilFinish();
-  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
index 528cfb06d47b..043fe939f132 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
@@ -32,20 +32,6 @@
  * @param <T> the type of the values being transcoded
  */
 public abstract class AtomicCoder<T> extends StructuredCoder<T> {
-  /**
-   * Returns an empty list.
-   *
-   * <p>{@link CoderFactories#fromStaticMethods(Class)} builds a {@link CoderFactory} from the
-   * {@code #of()} method and this method, used to determine the components of an object. Because
-   * {@link AtomicCoder} has no components, always returns an empty list.
-   *
-   * @param exampleValue unused, but part of the latent interface expected by {@link
-   *     CoderFactories#fromStaticMethods}
-   */
-  public static <T> List<Object> getInstanceComponents(T exampleValue) {
-    return Collections.emptyList();
-  }
-
   /**
    * {@inheritDoc}.
    *
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
index 2aa2b44c63d7..f82c065f488e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
@@ -34,6 +34,7 @@
 import java.util.SortedMap;
 import java.util.SortedSet;
 import javax.annotation.Nullable;
+import org.apache.avro.AvroRuntimeException;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericDatumWriter;
@@ -140,18 +141,39 @@ public static <T> AvroCoder<T> of(Class<T> type, Schema schema) {
     return new AvroCoder<>(type, schema);
   }
 
-  public static final CoderProvider PROVIDER = new CoderProvider() {
+  /**
+   * Returns a {@link CoderProvider} which uses the {@link AvroCoder} if possible for
+   * all types.
+   *
+   * <p>It is unsafe to register this as a {@link CoderProvider} because Avro will reflectively
+   * accept dangerous types such as {@link Object}.
+   *
+   * <p>This method is invoked reflectively from {@link DefaultCoder}.
+   */
+  @SuppressWarnings("unused")
+  public static CoderProvider getCoderProvider() {
+    return new AvroCoderProvider();
+  }
+
+  /**
+   * A {@link CoderProvider} that constructs an {@link AvroCoder} for Avro compatible classes.
+   *
+   * <p>It is unsafe to register this as a {@link CoderProvider} because Avro will reflectively
+   * accept dangerous types such as {@link Object}.
+   */
+  static class AvroCoderProvider extends CoderProvider {
     @Override
-    public <T> Coder<T> getCoder(TypeDescriptor<T> typeDescriptor) {
-      // This is a downcast from `? super T` to T. However, because
-      // it comes from a TypeDescriptor<T>, the class object itself
-      // is the same so the supertype in question shares the same
-      // generated AvroCoder schema.
-      @SuppressWarnings("unchecked")
-      Class<T> rawType = (Class<T>) typeDescriptor.getRawType();
-      return AvroCoder.of(rawType);
+    public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+        List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+      try {
+        return AvroCoder.of(typeDescriptor);
+      } catch (AvroRuntimeException e) {
+        throw new CannotProvideCoderException(
+            String.format("%s is not compatible with Avro", typeDescriptor),
+            e);
+      }
     }
-  };
+  }
 
   private final Class<T> type;
   private final SerializableSchemaSupplier schemaSupplier;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
index 28cb627017f6..d83d8328e73e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.List;
 import org.apache.beam.sdk.util.ExposedByteArrayOutputStream;
 import org.apache.beam.sdk.util.StreamUtils;
@@ -45,13 +44,6 @@ public static ByteArrayCoder of() {
     return INSTANCE;
   }
 
-  /**
-   * Returns an empty list. {@link ByteArrayCoder} has no components.
-   */
-  public static <T> List<Object> getInstanceComponents(T ignored) {
-    return Collections.emptyList();
-  }
-
   /////////////////////////////////////////////////////////////////////////////
 
   private static final ByteArrayCoder INSTANCE = new ByteArrayCoder();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CannotProvideCoderException.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CannotProvideCoderException.java
index c37ec00e4f02..bc2ef3fe2dde 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CannotProvideCoderException.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CannotProvideCoderException.java
@@ -18,7 +18,7 @@
 package org.apache.beam.sdk.coders;
 
 /**
- * The exception thrown when a {@link CoderProvider} cannot
+ * The exception thrown when a {@link CoderRegistry} or {@link CoderProvider} cannot
  * provide a {@link Coder} that has been requested.
  */
 public class CannotProvideCoderException extends Exception {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index 41e83acf6e50..eeafbd228603 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -51,9 +51,9 @@
  *
  * <p>{@link Coder} classes for compound types are often composed from coder classes for types
  * contains therein. The composition of {@link Coder} instances into a coder for the compound
- * class is the subject of the {@link CoderFactory} type, which enables automatic generic
+ * class is the subject of the {@link CoderProvider} type, which enables automatic generic
  * composition of {@link Coder} classes within the {@link CoderRegistry}. With particular
- * static methods on a compound {@link Coder} class, a {@link CoderFactory} can be automatically
+ * static methods on a compound {@link Coder} class, a {@link CoderProvider} can be automatically
  * inferred. See {@link KvCoder} for an example of a simple compound {@link Coder} that supports
  * automatic composition in the {@link CoderRegistry}.
  *
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
deleted file mode 100644
index 4f05c95f0b2c..000000000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactories.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.coders;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.base.MoreObjects;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.ParameterizedType;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.apache.beam.sdk.values.TypeDescriptor;
-
-/**
- * Static utility methods for creating and working with {@link Coder}s.
- */
-public final class CoderFactories {
-  private CoderFactories() { } // Static utility class
-
-  /**
-   * Creates a {@link CoderFactory} built from particular static methods of a class that
-   * implements {@link Coder}.
-   *
-   * <p>The class must have the following static methods:
-   *
-   * <ul>
-   * <li> {@code
-   * public static Coder<T> of(Coder<X> argCoder1, Coder<Y> argCoder2, ...)
-   * }
-   * <li> {@code
-   * public static List<Object> getInstanceComponents(T exampleValue);
-   * }
-   * </ul>
-   *
-   * <p>The {@code of(...)} method will be used to construct a
-   * {@code Coder<T>} from component {@link Coder}s.
-   * It must accept one {@link Coder} argument for each
-   * generic type parameter of {@code T}. If {@code T} takes no generic
-   * type parameters, then the {@code of()} factory method should take
-   * no arguments.
-   *
-   * <p>The {@code getInstanceComponents} method will be used to
-   * decompose a value during the {@link Coder} inference process,
-   * to automatically choose coders for the components.
-   *
-   * <p>Note that the class {@code T} to be coded may be a
-   * not-yet-specialized generic class.
-   * For a generic class {@code MyClass<X>} and an actual type parameter
-   * {@code Foo}, the {@link CoderFactoryFromStaticMethods} will
-   * accept any {@code Coder<Foo>} and produce a {@code Coder<MyClass<Foo>>}.
-   *
-   * <p>For example, the {@link CoderFactory} returned by
-   * {@code fromStaticMethods(ListCoder.class)}
-   * will produce a {@code Coder<List<X>>} for any {@code Coder Coder<X>}.
-   */
-  public static <T> CoderFactory fromStaticMethods(Class<T> clazz) {
-    checkArgument(
-        Coder.class.isAssignableFrom(clazz),
-        "%s is not a subtype of %s",
-        clazz.getName(),
-        Coder.class.getSimpleName());
-    return new CoderFactoryFromStaticMethods((Class<? extends Coder>) clazz);
-  }
-
-  /**
-   * Creates a {@link CoderFactory} that always returns the
-   * given coder.
-   *
-   * <p>The {@code getInstanceComponents} method of this
-   * {@link CoderFactory} always returns an empty list.
-   */
-  public static <T> CoderFactory forCoder(Coder<T> coder) {
-    return new CoderFactoryForCoder<>(coder);
-  }
-
-  /**
-   * See {@link #fromStaticMethods} for a detailed description
-   * of the characteristics of this {@link CoderFactory}.
-   */
-  private static class CoderFactoryFromStaticMethods implements CoderFactory {
-
-    @Override
-    @SuppressWarnings("rawtypes")
-    public Coder<?> create(List<? extends Coder<?>> componentCoders) {
-      try {
-        return (Coder) factoryMethod.invoke(
-            null /* static */, componentCoders.toArray());
-      } catch (IllegalAccessException
-           | IllegalArgumentException
-           | InvocationTargetException
-           | NullPointerException
-           | ExceptionInInitializerError exn) {
-        throw new IllegalStateException(
-            "error when invoking Coder factory method " + factoryMethod,
-            exn);
-      }
-    }
-
-    @Override
-    public List<Object> getInstanceComponents(Object value) {
-      try {
-        @SuppressWarnings("unchecked")
-        List<Object> components =  (List<Object>) getComponentsMethod.invoke(
-            null /* static */, value);
-        return components;
-      } catch (IllegalAccessException
-          | IllegalArgumentException
-          | InvocationTargetException
-          | NullPointerException
-          | ExceptionInInitializerError exn) {
-        throw new IllegalStateException(
-            "error when invoking Coder getComponents method " + getComponentsMethod,
-            exn);
-      }
-    }
-
-    ////////////////////////////////////////////////////////////////////////////////
-
-    // Method to create a coder given component coders
-    // For a Coder class of kind * -> * -> ... n times ... -> *
-    // this has type Coder<?> -> Coder<?> -> ... n times ... -> Coder<T>
-    private final Method factoryMethod;
-
-    // Method to decompose a value of type T into its parts.
-    // For a Coder class of kind * -> * -> ... n times ... -> *
-    // this has type T -> List<Object>
-    // where the list has n elements.
-    private final Method getComponentsMethod;
-
-    /**
-     * Returns a CoderFactory that invokes the given static factory method
-     * to create the Coder.
-     */
-    private CoderFactoryFromStaticMethods(Class<? extends Coder> coderClazz) {
-      this.factoryMethod = getFactoryMethod(coderClazz);
-      this.getComponentsMethod = getInstanceComponentsMethod(coderClazz);
-    }
-
-    /**
-     * Returns the static {@code of} constructor method on {@code coderClazz}
-     * if it exists. It is assumed to have one {@link Coder} parameter for
-     * each type parameter of {@code coderClazz}.
-     */
-    private Method getFactoryMethod(Class<?> coderClazz) {
-      Method factoryMethodCandidate;
-
-      // Find the static factory method of coderClazz named 'of' with
-      // the appropriate number of type parameters.
-      int numTypeParameters = coderClazz.getTypeParameters().length;
-      Class<?>[] factoryMethodArgTypes = new Class<?>[numTypeParameters];
-      Arrays.fill(factoryMethodArgTypes, Coder.class);
-      try {
-        factoryMethodCandidate =
-            coderClazz.getDeclaredMethod("of", factoryMethodArgTypes);
-      } catch (NoSuchMethodException | SecurityException exn) {
-        throw new IllegalArgumentException(
-            "cannot register Coder " + coderClazz + ": "
-            + "does not have an accessible method named 'of' with "
-            + numTypeParameters + " arguments of Coder type",
-            exn);
-      }
-      if (!Modifier.isStatic(factoryMethodCandidate.getModifiers())) {
-        throw new IllegalArgumentException(
-            "cannot register Coder " + coderClazz + ": "
-            + "method named 'of' with " + numTypeParameters
-            + " arguments of Coder type is not static");
-      }
-      if (!coderClazz.isAssignableFrom(factoryMethodCandidate.getReturnType())) {
-        throw new IllegalArgumentException(
-            "cannot register Coder " + coderClazz + ": "
-            + "method named 'of' with " + numTypeParameters
-            + " arguments of Coder type does not return a " + coderClazz);
-      }
-      try {
-        if (!factoryMethodCandidate.isAccessible()) {
-          factoryMethodCandidate.setAccessible(true);
-        }
-      } catch (SecurityException exn) {
-        throw new IllegalArgumentException(
-            "cannot register Coder " + coderClazz + ": "
-            + "method named 'of' with " + numTypeParameters
-            + " arguments of Coder type is not accessible",
-            exn);
-      }
-
-      return factoryMethodCandidate;
-    }
-
-    /**
-     * Finds the static method on {@code coderType} to use
-     * to decompose a value of type {@code T} into components,
-     * each corresponding to an argument of the {@code of}
-     * method.
-     */
-    private <T, CoderT extends Coder> Method getInstanceComponentsMethod(Class<CoderT> coderClazz) {
-      TypeDescriptor<CoderT> coderType = TypeDescriptor.of(coderClazz);
-      TypeDescriptor<T> argumentType = getCodedType(coderType);
-
-      // getInstanceComponents may be implemented in a superclass,
-      // so we search them all for an applicable method. We do not
-      // try to be clever about finding the best overload. It may
-      // be in a generic superclass, erased to accept an Object.
-      // However, subtypes are listed before supertypes (it is a
-      // topological ordering) so probably the best one will be chosen
-      // if there are more than one (which should be rare)
-      for (TypeDescriptor<?> supertype : coderType.getClasses()) {
-        for (Method method : supertype.getRawType().getDeclaredMethods()) {
-          if (method.getName().equals("getInstanceComponents")) {
-            TypeDescriptor<?> formalArgumentType = supertype.getArgumentTypes(method).get(0);
-            if (formalArgumentType.getRawType().isAssignableFrom(argumentType.getRawType())) {
-              return method;
-            }
-          }
-        }
-      }
-
-      throw new IllegalArgumentException(
-          "cannot create a CoderFactory from " + coderType + ": "
-          + "does not have an accessible method "
-          + "'getInstanceComponents'");
-    }
-
-    /**
-     * If {@code coderType} is a subclass of {@link Coder} for a specific type {@code T}, returns
-     * {@code T.class}. Otherwise, raises IllegalArgumentException.
-     */
-    private <T, CoderT extends Coder> TypeDescriptor<T> getCodedType(
-        TypeDescriptor<CoderT> coderType) {
-      TypeDescriptor<? super CoderT> coderSupertype = coderType.getSupertype(Coder.class);
-      ParameterizedType coderIface = (ParameterizedType) coderSupertype.getType();
-      @SuppressWarnings("unchecked")
-      TypeDescriptor<T> token =
-          (TypeDescriptor<T>) TypeDescriptor.of(coderIface.getActualTypeArguments()[0]);
-      return token;
-    }
-
-    @Override
-    public String toString() {
-      return MoreObjects.toStringHelper(getClass())
-          .add("factoryMethod", factoryMethod)
-          .add("getComponentsMethod", getComponentsMethod)
-          .toString();
-    }
-  }
-
-  /**
-   * See {@link #forCoder} for a detailed description of this
-   * {@link CoderFactory}.
-   */
-  private static class CoderFactoryForCoder<T> implements CoderFactory {
-    private final Coder<T> coder;
-
-    public CoderFactoryForCoder(Coder<T> coder) {
-      this.coder = coder;
-    }
-
-    @Override
-    public Coder<?> create(List<? extends Coder<?>> componentCoders) {
-      return this.coder;
-    }
-
-    @Override
-    public List<Object> getInstanceComponents(Object value) {
-      return Collections.emptyList();
-    }
-
-    @Override
-    public String toString() {
-      return MoreObjects.toStringHelper(getClass())
-          .add("coder", coder)
-          .toString();
-    }
-  }
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactory.java
deleted file mode 100644
index 22d03fa2ec00..000000000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderFactory.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.coders;
-
-import java.util.List;
-
-/**
- * A {@link CoderFactory} creates coders and decomposes values.
- * It may operate on a parameterized type, such as {@link List},
- * in which case the {@link #create} method accepts a list of
- * coders to use for the type parameters.
- */
-public interface CoderFactory {
-
-  /**
-   * Returns a {@code Coder<?>}, given argument coder to use for
-   * values of a particular type, given the Coders for each of
-   * the type's generic parameter types.
-   */
-  Coder<?> create(List<? extends Coder<?>> componentCoders);
-
-  /**
-   * Returns a list of objects contained in {@code value}, one per
-   * type argument, or {@code null} if none can be determined.
-   * The list of returned objects should be the same size as the
-   * list of coders required by {@link #create}.
-   */
-  List<Object> getInstanceComponents(Object value);
-}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProvider.java
index 0db73eb6f17f..ac042ce54df5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProvider.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProvider.java
@@ -17,18 +17,25 @@
  */
 package org.apache.beam.sdk.coders;
 
+import java.util.List;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
- * A {@link CoderProvider} may create a {@link Coder} for
- * any concrete class.
+ * A {@link CoderProvider} provides {@link Coder}s.
+ *
+ * <p>It may operate on a parameterized type, such as {@link List}, in which case the
+ * {@link #coderFor} method accepts a list of coders to use for the type parameters.
  */
-public interface CoderProvider {
+public abstract class CoderProvider {
 
   /**
-   * Provides a coder for a given class, if possible.
+   * Returns a {@code Coder<T>} to use for values of a particular type, given the Coders for each of
+   * the type's generic parameter types.
    *
-   * @throws CannotProvideCoderException if no coder can be provided
+   * <p>Throws {@link CannotProvideCoderException} if this {@link CoderProvider} cannot provide
+   * a coder for this type and components.
    */
-  <T> Coder<T> getCoder(TypeDescriptor<T> type) throws CannotProvideCoderException;
+  public abstract <T> Coder<T> coderFor(
+      TypeDescriptor<T> typeDescriptor, List<? extends Coder<?>> componentCoders)
+      throws CannotProvideCoderException;
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviderRegistrar.java
similarity index 76%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistrar.java
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviderRegistrar.java
index fced97657e4a..35d061d78e1e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistrar.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviderRegistrar.java
@@ -18,7 +18,7 @@
 package org.apache.beam.sdk.coders;
 
 import com.google.auto.service.AutoService;
-import java.util.Map;
+import java.util.List;
 import java.util.ServiceLoader;
 import org.apache.beam.sdk.annotations.Experimental;
 
@@ -31,15 +31,12 @@
  * {@link AutoService} to generate the necessary META-INF files automatically.
  */
 @Experimental
-public interface CoderRegistrar {
+public interface CoderProviderRegistrar {
   /**
-   * Returns a mapping of {@link Class classes} to {@link CoderFactory coder factories} which
+   * Returns a list of {@link CoderProvider coder providers} which
    * will be registered by default within each {@link CoderRegistry coder registry} instance.
    *
-   * <p>See {@link CoderFactories} for convenience methods to construct a {@link CoderFactory}.
-   *
-   * <p>Note that a warning is logged if multiple {@link CoderRegistrar coder registrars} provide
-   * mappings for the same {@link Class}.
+   * <p>See {@link CoderProviders} for convenience methods to construct a {@link CoderProvider}.
    */
-  Map<Class<?>, CoderFactory> getCoderFactoriesToUseForClasses();
+  List<CoderProvider> getCoderProviders();
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java
index c072008d5f28..414fd8bb6880 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java
@@ -19,146 +19,178 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
+import com.google.common.base.MoreObjects;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
 import java.util.List;
-import org.apache.beam.sdk.util.InstanceBuilder;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
- * Static utility methods for working with {@link CoderProvider CoderProviders}.
+ * Static utility methods for creating and working with {@link CoderProvider}s.
  */
 public final class CoderProviders {
-
-  // Static utility class
-  private CoderProviders() { }
+  private CoderProviders() { } // Static utility class
 
   /**
-   * Creates a {@link CoderProvider} built from particular static methods of a class that
-   * implements {@link Coder}. The requirements for this method are precisely the requirements
-   * for a {@link Coder} class to be usable with {@link DefaultCoder} annotations.
-   *
-   * <p>The class must have the following static method:
-   *
-   * <pre>{@code
-   * public static Coder<T> of(TypeDescriptor<T> type)
-   * }
-   * </pre>
+   * Creates a {@link CoderProvider} from a class's
+   * {@code static <T> Coder<T> of(TypeDescriptor<T>, List<Coder<?>>}) method.
    */
-  public static <T> CoderProvider fromStaticMethods(Class<T> clazz) {
-    return new CoderProviderFromStaticMethods(clazz);
+  public static CoderProvider fromStaticMethods(Class<?> rawType, Class<?> coderClazz) {
+    checkArgument(
+        Coder.class.isAssignableFrom(coderClazz),
+        "%s is not a subtype of %s",
+        coderClazz.getName(),
+        Coder.class.getSimpleName());
+    return new CoderProviderFromStaticMethods(rawType, coderClazz);
   }
 
-
   /**
-   * Returns a {@link CoderProvider} that consults each of the provider {@code coderProviders}
-   * and returns the first {@link Coder} provided.
-   *
-   * <p>Note that the order in which the providers are listed matters: While the set of types
-   * handled will be the union of those handled by all of the providers in the list, the actual
-   * {@link Coder} provided by the first successful provider may differ, and may have inferior
-   * properties. For example, not all {@link Coder Coders} are deterministic, handle {@code null}
-   * values, or have comparable performance.
+   * Creates a {@link CoderProvider} that always returns the
+   * given coder for the specified type.
    */
-  public static CoderProvider firstOf(CoderProvider... coderProviders) {
-    return new FirstOf(ImmutableList.copyOf(coderProviders));
+  public static CoderProvider forCoder(TypeDescriptor<?> type, Coder<?> coder) {
+    return new CoderProviderForCoder(type, coder);
   }
 
-  ///////////////////////////////////////////////////////////////////////////////////////////////
-
   /**
-   * @see #firstOf
+   * See {@link #fromStaticMethods} for a detailed description
+   * of the characteristics of this {@link CoderProvider}.
    */
-  private static class FirstOf implements CoderProvider {
-
-    private Iterable<CoderProvider> providers;
-
-    public FirstOf(Iterable<CoderProvider> providers) {
-      this.providers = providers;
-    }
+  private static class CoderProviderFromStaticMethods extends CoderProvider {
 
     @Override
-    public <T> Coder<T> getCoder(TypeDescriptor<T> type) throws CannotProvideCoderException {
-      List<String> messages = Lists.newArrayList();
-      for (CoderProvider provider : providers) {
-        try {
-          return provider.getCoder(type);
-        } catch (CannotProvideCoderException exc) {
-          messages.add(String.format("%s could not provide a Coder for type %s: %s",
-              provider, type, exc.getMessage()));
-        }
+    public <T> Coder<T> coderFor(TypeDescriptor<T> type, List<? extends Coder<?>> componentCoders)
+        throws CannotProvideCoderException {
+      if (!this.rawType.equals(type.getRawType())) {
+        throw new CannotProvideCoderException(String.format(
+            "Unable to provide coder for %s, this factory can only provide coders for %s",
+            type,
+            this.rawType));
+      }
+      try {
+        return (Coder) factoryMethod.invoke(
+            null /* static */, componentCoders.toArray());
+      } catch (IllegalAccessException
+           | IllegalArgumentException
+           | InvocationTargetException
+           | NullPointerException
+           | ExceptionInInitializerError exn) {
+        throw new IllegalStateException(
+            "error when invoking Coder factory method " + factoryMethod,
+            exn);
       }
-      throw new CannotProvideCoderException(
-          String.format("Cannot provide coder for type %s: %s.",
-              type, Joiner.on("; ").join(messages)));
     }
-  }
 
-  private static class CoderProviderFromStaticMethods implements CoderProvider {
+    ////////////////////////////////////////////////////////////////////////////////
 
-    /** If true, then clazz has {@code of(TypeDescriptor)}. If false, {@code of(Class)}. */
-    private final boolean takesTypeDescriptor;
-    private final Class<?> clazz;
+    // Type raw type used to filter the incoming type on.
+    private final Class<?> rawType;
 
-    public CoderProviderFromStaticMethods(Class<?> clazz) {
-      // Note that the second condition supports older classes, which only needed to provide
-      // of(Class), not of(TypeDescriptor). Our own classes have updated to accept a
-      // TypeDescriptor. Hence the error message points only to the current specification,
-      // not both acceptable conditions.
-      checkArgument(classTakesTypeDescriptor(clazz) || classTakesClass(clazz),
-          "Class " + clazz.getCanonicalName()
-          + " is missing required static method of(TypeDescriptor).");
+    // Method to create a coder given component coders
+    // For a Coder class of kind * -> * -> ... n times ... -> *
+    // this has type Coder<?> -> Coder<?> -> ... n times ... -> Coder<T>
+    private final Method factoryMethod;
 
-      this.takesTypeDescriptor = classTakesTypeDescriptor(clazz);
-      this.clazz = clazz;
+    /**
+     * Returns a CoderProvider that invokes the given static factory method
+     * to create the Coder.
+     */
+    private CoderProviderFromStaticMethods(Class<?> rawType, Class<?> coderClazz) {
+      this.rawType = rawType;
+      this.factoryMethod = getFactoryMethod(coderClazz);
     }
 
-    @Override
-    public <T> Coder<T> getCoder(TypeDescriptor<T> type) throws CannotProvideCoderException {
+    /**
+     * Returns the static {@code of} constructor method on {@code coderClazz}
+     * if it exists. It is assumed to have one {@link Coder} parameter for
+     * each type parameter of {@code coderClazz}.
+     */
+    private Method getFactoryMethod(Class<?> coderClazz) {
+      Method factoryMethodCandidate;
+
+      // Find the static factory method of coderClazz named 'of' with
+      // the appropriate number of type parameters.
+      int numTypeParameters = coderClazz.getTypeParameters().length;
+      Class<?>[] factoryMethodArgTypes = new Class<?>[numTypeParameters];
+      Arrays.fill(factoryMethodArgTypes, Coder.class);
       try {
-        if (takesTypeDescriptor) {
-          @SuppressWarnings("unchecked")
-          Coder<T> result = InstanceBuilder.ofType(Coder.class)
-              .fromClass(clazz)
-              .fromFactoryMethod("of")
-              .withArg(TypeDescriptor.class, type)
-              .build();
-          return result;
-        } else {
-          @SuppressWarnings("unchecked")
-          Coder<T> result = InstanceBuilder.ofType(Coder.class)
-              .fromClass(clazz)
-              .fromFactoryMethod("of")
-              .withArg(Class.class, type.getRawType())
-              .build();
-          return result;
-        }
-      } catch (RuntimeException exc) {
-        if (exc.getCause() instanceof InvocationTargetException) {
-          throw new CannotProvideCoderException(exc.getCause().getCause());
+        factoryMethodCandidate =
+            coderClazz.getDeclaredMethod("of", factoryMethodArgTypes);
+      } catch (NoSuchMethodException | SecurityException exn) {
+        throw new IllegalArgumentException(
+            "cannot register Coder " + coderClazz + ": "
+            + "does not have an accessible method named 'of' with "
+            + numTypeParameters + " arguments of Coder type",
+            exn);
+      }
+      if (!Modifier.isStatic(factoryMethodCandidate.getModifiers())) {
+        throw new IllegalArgumentException(
+            "cannot register Coder " + coderClazz + ": "
+            + "method named 'of' with " + numTypeParameters
+            + " arguments of Coder type is not static");
+      }
+      if (!coderClazz.isAssignableFrom(factoryMethodCandidate.getReturnType())) {
+        throw new IllegalArgumentException(
+            "cannot register Coder " + coderClazz + ": "
+            + "method named 'of' with " + numTypeParameters
+            + " arguments of Coder type does not return a " + coderClazz);
+      }
+      try {
+        if (!factoryMethodCandidate.isAccessible()) {
+          factoryMethodCandidate.setAccessible(true);
         }
-        throw exc;
+      } catch (SecurityException exn) {
+        throw new IllegalArgumentException(
+            "cannot register Coder " + coderClazz + ": "
+            + "method named 'of' with " + numTypeParameters
+            + " arguments of Coder type is not accessible",
+            exn);
       }
+
+      return factoryMethodCandidate;
     }
 
-    private boolean classTakesTypeDescriptor(Class<?> clazz) {
-      try {
-        clazz.getDeclaredMethod("of", TypeDescriptor.class);
-        return true;
-      } catch (NoSuchMethodException | SecurityException exc) {
-        return false;
-      }
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(getClass())
+          .add("rawType", rawType)
+          .add("factoryMethod", factoryMethod)
+          .toString();
     }
+  }
 
-    private boolean classTakesClass(Class<?> clazz) {
-      try {
-        clazz.getDeclaredMethod("of", Class.class);
-        return true;
-      } catch (NoSuchMethodException | SecurityException exc) {
-        return false;
+  /**
+   * See {@link #forCoder} for a detailed description of this {@link CoderProvider}.
+   */
+  private static class CoderProviderForCoder extends CoderProvider {
+    private final Coder<?> coder;
+    private final TypeDescriptor<?> type;
+
+    public CoderProviderForCoder(TypeDescriptor<?> type, Coder<?> coder){
+      this.type = type;
+      this.coder = coder;
+    }
+
+    @Override
+    public <T> Coder<T> coderFor(TypeDescriptor<T> type, List<? extends Coder<?>> componentCoders)
+        throws CannotProvideCoderException {
+      if (!this.type.equals(type)) {
+        throw new CannotProvideCoderException(String.format(
+            "Unable to provide coder for %s, this factory can only provide coders for %s",
+            type,
+            this.type));
       }
+      return (Coder) coder;
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(getClass())
+          .add("type", type)
+          .add("coder", coder)
+          .toString();
     }
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
index 7f0f632a04af..2ba548aa5869 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
@@ -20,12 +20,10 @@
 import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Predicate;
-import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
@@ -37,12 +35,13 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.ServiceLoader;
 import java.util.Set;
-import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.CannotProvideCoderException.ReasonCode;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -57,102 +56,118 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * A {@link CoderRegistry} allows registering the default {@link Coder} to use for a Java class,
- * and looking up and instantiating the default {@link Coder} for a Java type.
+ * A {@link CoderRegistry} allows creating a {@link Coder} for a given Java {@link Class class} or
+ * {@link TypeDescriptor type descriptor}.
  *
- * <p>{@link CoderRegistry} uses the following mechanisms to determine a default {@link Coder} for a
- * Java class, in order of precedence:
- * <ol>
- *   <li>Registration:
- *     <ul>
- *       <li>A {@link CoderFactory} can be registered to handle a particular class via
- *           {@link #registerCoder(Class, CoderFactory)}.</li>
- *       <li>A {@link Coder} class with the static methods to satisfy
- *           {@link CoderFactories#fromStaticMethods} can be registered via
- *           {@link #registerCoder(Class, Class)}.</li>
- *       <li>Types can be automatically registered via {@link CoderRegistrar coder registrars}.</li>
- *     </ul>
- *   <li>Annotations: {@link DefaultCoder} can be used to annotate a type with
- *       the default {@code Coder} type. The {@link Coder} class must satisfy the requirements
- *       of {@link CoderProviders#fromStaticMethods}.
- *   <li>Fallback: A fallback {@link CoderProvider} is used to attempt to provide a {@link Coder}
- *       for any type. By default, there is {@link SerializableCoder#PROVIDER}, which can provide a
- *       {@link Coder} for any type that is serializable via Java serialization. The fallback
- *       {@link CoderProvider} can be get and set respectively using
- *       {@link #getFallbackCoderProvider()} and {@link #setFallbackCoderProvider}. Multiple
- *       fallbacks can be chained together using {@link CoderProviders#firstOf}.
- * </ol>
+ * <p>Creation of the {@link Coder} is delegated to one of the many registered
+ * {@link CoderProvider coder providers} based upon the registration order.
+ *
+ * <p>By default, the {@link CoderProvider coder provider} precedence order is as follows:
+ * <ul>
+ *   <li>Coder providers registered programmatically with
+ *       {@link CoderRegistry#registerCoderProvider(CoderProvider)}.</li>
+ *   <li>A default coder provider for common Java (Byte, Double, List, ...) and
+ *       Apache Beam (KV, ...) types.</li>
+ *   <li>Coder providers registered automatically through a {@link CoderProviderRegistrar} using
+ *       a {@link ServiceLoader}. Note that the {@link ServiceLoader} registration order is
+ *       consistent but may change due to the addition or removal of libraries exposed
+ *       to the application. This can impact the coder returned if multiple coder providers
+ *       are capable of supplying a coder for the specified type.</li>
+ * </ul>
+ *
+ * <p>Note that if multiple {@link CoderProvider coder providers} can provide a {@link Coder} for
+ * a given type, the precedence order above defines which {@link CoderProvider} is chosen.
  */
-public class CoderRegistry implements CoderProvider {
+public class CoderRegistry {
 
   private static final Logger LOG = LoggerFactory.getLogger(CoderRegistry.class);
-  private static final Map<Class<?>, CoderFactory> REGISTERED_CODER_FACTORIES_PER_CLASS;
+  private static final List<CoderProvider> REGISTERED_CODER_FACTORIES;
+
+  /** A {@link CoderProvider} for common Java SDK and Apache Beam SDK types. */
+  private static class CommonTypes extends CoderProvider {
+    private final Map<Class<?>, CoderProvider> commonTypesToCoderProviders;
+
+    private CommonTypes() {
+      ImmutableMap.Builder<Class<?>, CoderProvider> builder = ImmutableMap.builder();
+      builder.put(Byte.class,
+          CoderProviders.fromStaticMethods(Byte.class, ByteCoder.class));
+      builder.put(BitSet.class,
+          CoderProviders.fromStaticMethods(BitSet.class, BitSetCoder.class));
+      builder.put(Double.class,
+          CoderProviders.fromStaticMethods(Double.class, DoubleCoder.class));
+      builder.put(Instant.class,
+          CoderProviders.fromStaticMethods(Instant.class, InstantCoder.class));
+      builder.put(Integer.class,
+          CoderProviders.fromStaticMethods(Integer.class, VarIntCoder.class));
+      builder.put(Iterable.class,
+          CoderProviders.fromStaticMethods(Iterable.class, IterableCoder.class));
+      builder.put(KV.class,
+          CoderProviders.fromStaticMethods(KV.class, KvCoder.class));
+      builder.put(List.class,
+          CoderProviders.fromStaticMethods(List.class, ListCoder.class));
+      builder.put(Long.class,
+          CoderProviders.fromStaticMethods(Long.class, VarLongCoder.class));
+      builder.put(Map.class,
+          CoderProviders.fromStaticMethods(Map.class, MapCoder.class));
+      builder.put(Set.class,
+          CoderProviders.fromStaticMethods(Set.class, SetCoder.class));
+      builder.put(String.class,
+          CoderProviders.fromStaticMethods(String.class, StringUtf8Coder.class));
+      builder.put(TimestampedValue.class,
+          CoderProviders.fromStaticMethods(
+              TimestampedValue.class, TimestampedValue.TimestampedValueCoder.class));
+      builder.put(Void.class,
+          CoderProviders.fromStaticMethods(Void.class, VoidCoder.class));
+      builder.put(byte[].class,
+          CoderProviders.fromStaticMethods(byte[].class, ByteArrayCoder.class));
+      builder.put(IntervalWindow.class,
+          CoderProviders.forCoder(
+              TypeDescriptor.of(IntervalWindow.class), IntervalWindow.getCoder()));
+      commonTypesToCoderProviders = builder.build();
+    }
+
+    @Override
+    public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+        List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+      CoderProvider factory = commonTypesToCoderProviders.get(typeDescriptor.getRawType());
+      if (factory == null) {
+        throw new CannotProvideCoderException(
+            String.format("%s is not one of the common types.", typeDescriptor));
+      }
+      return factory.coderFor(typeDescriptor, componentCoders);
+    }
+  }
 
   static {
-    // Register the standard coders first so they are chosen as the default
-    Multimap<Class<?>, CoderFactory> codersToRegister = HashMultimap.create();
-    codersToRegister.put(Byte.class, CoderFactories.fromStaticMethods(ByteCoder.class));
-    codersToRegister.put(BitSet.class, CoderFactories.fromStaticMethods(BitSetCoder.class));
-    codersToRegister.put(Double.class, CoderFactories.fromStaticMethods(DoubleCoder.class));
-    codersToRegister.put(Instant.class, CoderFactories.fromStaticMethods(InstantCoder.class));
-    codersToRegister.put(Integer.class, CoderFactories.fromStaticMethods(VarIntCoder.class));
-    codersToRegister.put(Iterable.class, CoderFactories.fromStaticMethods(IterableCoder.class));
-    codersToRegister.put(KV.class, CoderFactories.fromStaticMethods(KvCoder.class));
-    codersToRegister.put(List.class, CoderFactories.fromStaticMethods(ListCoder.class));
-    codersToRegister.put(Long.class, CoderFactories.fromStaticMethods(VarLongCoder.class));
-    codersToRegister.put(Map.class, CoderFactories.fromStaticMethods(MapCoder.class));
-    codersToRegister.put(Set.class, CoderFactories.fromStaticMethods(SetCoder.class));
-    codersToRegister.put(String.class, CoderFactories.fromStaticMethods(StringUtf8Coder.class));
-    codersToRegister.put(TimestampedValue.class,
-        CoderFactories.fromStaticMethods(TimestampedValue.TimestampedValueCoder.class));
-    codersToRegister.put(Void.class, CoderFactories.fromStaticMethods(VoidCoder.class));
-    codersToRegister.put(byte[].class, CoderFactories.fromStaticMethods(ByteArrayCoder.class));
-    codersToRegister.put(IntervalWindow.class, CoderFactories.forCoder(IntervalWindow.getCoder()));
+    // Register the standard coders first so they are chosen over ServiceLoader ones
+    List<CoderProvider> codersToRegister = new ArrayList<>();
+    codersToRegister.add(new CommonTypes());
 
     // Enumerate all the CoderRegistrars in a deterministic order, adding all coders to register
-    Set<CoderRegistrar> registrars = Sets.newTreeSet(ObjectsClassComparator.INSTANCE);
+    Set<CoderProviderRegistrar> registrars = Sets.newTreeSet(ObjectsClassComparator.INSTANCE);
     registrars.addAll(Lists.newArrayList(
-        ServiceLoader.load(CoderRegistrar.class, ReflectHelpers.findClassLoader())));
-    for (CoderRegistrar registrar : registrars) {
-      for (Map.Entry<Class<?>, CoderFactory> entry
-          : registrar.getCoderFactoriesToUseForClasses().entrySet()) {
-        codersToRegister.put(entry.getKey(), entry.getValue());
-      }
-    }
-
-    // Warn the user if multiple coders want to be registered for the same class
-    Map<Class<?>, Collection<CoderFactory>> multipleRegistrations =
-        Maps.filterValues(codersToRegister.asMap(), new Predicate<Collection<CoderFactory>>() {
-      @Override
-      public boolean apply(@Nonnull Collection<CoderFactory> input) {
-        return input.size() > 1;
-      }
-    });
-    for (Map.Entry<Class<?>, Collection<CoderFactory>> entry : multipleRegistrations.entrySet()) {
-      LOG.warn("Multiple CoderFactory registrations {} found for class {}, using {}.",
-          entry.getKey(), entry.getValue(), entry.getValue().iterator().next());
+        ServiceLoader.load(CoderProviderRegistrar.class, ReflectHelpers.findClassLoader())));
+    for (CoderProviderRegistrar registrar : registrars) {
+        codersToRegister.addAll(registrar.getCoderProviders());
     }
 
-    // Build a map choosing the first coder within the multimap as the default
-    ImmutableMap.Builder<Class<?>, CoderFactory> registeredCoderFactoriesPerClassBuilder =
-        ImmutableMap.builder();
-    for (Map.Entry<Class<?>, Collection<CoderFactory>> entry
-        : codersToRegister.asMap().entrySet()) {
-      registeredCoderFactoriesPerClassBuilder.put(
-          entry.getKey(), entry.getValue().iterator().next());
-    }
-    REGISTERED_CODER_FACTORIES_PER_CLASS = registeredCoderFactoriesPerClassBuilder.build();
+    REGISTERED_CODER_FACTORIES = ImmutableList.copyOf(codersToRegister);
   }
 
   /**
    * Creates a CoderRegistry containing registrations for all standard coders part of the core Java
-   * Apache Beam SDK and also any registrations provided by {@link CoderRegistrar coder registrars}.
+   * Apache Beam SDK and also any registrations provided by
+   * {@link CoderProviderRegistrar coder registrars}.
    *
-   * <p>Multiple registrations for the same class result in the (in order of precedence):
+   * <p>Multiple registrations which can produce a coder for a given type result in a Coder created
+   * by the (in order of precedence):
    * <ul>
-   *   <li>Standard coder part of the core Apache Beam Java SDK being used.</li>
-   *   <li>The coder from the {@link CoderRegistrar} with the lexicographically smallest
-   *   {@link Class#getName() class name} being used.</li>
+   *   <li>{@link CoderProvider coder providers} registered programmatically through
+   *   {@link CoderRegistry#registerCoderProvider}.</li>
+   *   <li>{@link CoderProvider coder providers} for core types found within the Apache Beam Java
+   *   SDK being used.</li>
+   *   <li>The {@link CoderProvider coder providers} from the {@link CoderProviderRegistrar}
+   *   with the lexicographically smallest {@link Class#getName() class name} being used.</li>
    * </ul>
    */
   public static CoderRegistry createDefault() {
@@ -160,97 +175,69 @@ public static CoderRegistry createDefault() {
   }
 
   private CoderRegistry() {
-    coderFactoryMap = new HashMap<>(REGISTERED_CODER_FACTORIES_PER_CLASS);
-    setFallbackCoderProvider(
-        CoderProviders.firstOf(SerializableCoder.PROVIDER));
+    coderProviders = new LinkedList<>(REGISTERED_CODER_FACTORIES);
   }
 
   /**
-   * Registers {@code coderClazz} as the default {@link Coder} class to handle encoding and
-   * decoding instances of {@code clazz}, overriding prior registrations if any exist.
-   *
-   * <p>Supposing {@code T} is the static type corresponding to the {@code clazz}, then
-   * {@code coderClazz} should have a static factory method with the following signature:
-   *
-   * <pre> {@code
-   * public static Coder<T> of(Coder<X> argCoder1, Coder<Y> argCoder2, ...)
-   * } </pre>
+   * Registers {@code coderProvider} as a potential {@link CoderProvider} which can produce
+   * {@code Coder} instances.
    *
-   * <p>This method will be called to create instances of {@code Coder<T>} for values of type
-   * {@code T}, passing Coders for each of the generic type parameters of {@code T}.  If {@code T}
-   * takes no generic type parameters, then the {@code of()} factory method should have no
-   * arguments.
+   * <p>This method prioritizes this {@link CoderProvider} over all prior registered coders.
    *
-   * <p>If {@code T} is a parameterized type, then it should additionally have a method with the
-   * following signature:
-   *
-   * <pre> {@code
-   * public static List<Object> getInstanceComponents(T exampleValue);
-   * } </pre>
-   *
-   * <p>This method will be called to decompose a value during the {@link Coder} inference process,
-   * to automatically choose {@link Coder Coders} for the components.
-   *
-   * @param clazz the class of objects to be encoded
-   * @param coderClazz a class with static factory methods to provide {@link Coder Coders}
+   * <p>See {@link CoderProviders} for common {@link CoderProvider} patterns.
    */
-  public void registerCoder(Class<?> clazz, Class<?> coderClazz) {
-    registerCoder(clazz, CoderFactories.fromStaticMethods(coderClazz));
+  public void registerCoderProvider(CoderProvider coderProvider) {
+    coderProviders.addFirst(coderProvider);
   }
 
   /**
-   * Registers {@code coderFactory} as the default {@link CoderFactory} to produce {@code Coder}
-   * instances to decode and encode instances of {@code clazz}. This will override prior
-   * registrations if any exist.
+   * Registers the provided {@link Coder} for the given class.
+   *
+   * <p>Note that this is equivalent to {@code registerCoderForType(TypeDescriptor.of(clazz))}. See
+   * {@link #registerCoderForType(TypeDescriptor, Coder)} for further details.
    */
-  public void registerCoder(Class<?> clazz, CoderFactory coderFactory) {
-    coderFactoryMap.put(clazz, coderFactory);
+  public void registerCoderForClass(Class<?> clazz, Coder<?> coder) {
+    registerCoderForType(TypeDescriptor.of(clazz), coder);
   }
 
   /**
-   * Register the provided {@link Coder} for encoding all values of the specified {@code Class}.
-   * This will override prior registrations if any exist.
+   * Registers the provided {@link Coder} for the given type.
    *
-   * <p>Not for use with generic rawtypes. Instead, register a {@link CoderFactory} via
-   * {@link #registerCoder(Class, CoderFactory)} or ensure your {@code Coder} class has the
-   * appropriate static methods and register it directly via {@link #registerCoder(Class, Class)}.
+   * <p>Note that this is equivalent to
+   * {@code registerCoderProvider(CoderProviders.forCoder(type, coder))}. See
+   * {@link #registerCoderProvider} and {@link CoderProviders#forCoder} for further details.
    */
-  public <T> void registerCoder(Class<T> rawClazz, Coder<T> coder) {
-    checkArgument(
-        rawClazz.getTypeParameters().length == 0,
-        "CoderRegistry.registerCoder(Class<T>, Coder<T>) may not be used "
-            + "with unspecialized generic classes");
-
-    CoderFactory factory = CoderFactories.forCoder(coder);
-    registerCoder(rawClazz, factory);
+  public void registerCoderForType(TypeDescriptor<?> type, Coder<?> coder) {
+    registerCoderProvider(CoderProviders.forCoder(type, coder));
   }
 
   /**
-   * Returns the {@link Coder} to use by default for values of the given type.
-   *
-   * @throws CannotProvideCoderException if there is no default Coder.
+   * Returns the {@link Coder} to use for values of the given class.
+
+   * @throws CannotProvideCoderException if a {@link Coder} cannot be provided
    */
-  public <T> Coder<T> getDefaultCoder(TypeDescriptor<T> typeDescriptor)
-      throws CannotProvideCoderException {
-    return getDefaultCoder(typeDescriptor, Collections.<Type, Coder<?>>emptyMap());
+  public <T> Coder<T> getCoder(Class<T> clazz) throws CannotProvideCoderException {
+    return getCoder(TypeDescriptor.of(clazz));
   }
 
   /**
-   * See {@link #getDefaultCoder(TypeDescriptor)}.
+   * Returns the {@link Coder} to use for values of the given type.
+   *
+   * @throws CannotProvideCoderException if a {@link Coder} cannot be provided
    */
-  @Override
-  public <T> Coder<T> getCoder(TypeDescriptor<T> typeDescriptor)
-      throws CannotProvideCoderException {
-    return getDefaultCoder(typeDescriptor);
+  public <T> Coder<T> getCoder(TypeDescriptor<T> type) throws CannotProvideCoderException {
+    return getCoderFromTypeDescriptor(type, Collections.<Type, Coder<?>>emptyMap());
   }
 
   /**
-   * Returns the {@link Coder} to use by default for values of the given type, where the given input
+   * Returns the {@link Coder} for values of the given type, where the given input
    * type uses the given {@link Coder}.
    *
-   * @throws CannotProvideCoderException if there is no default Coder.
+   * @throws CannotProvideCoderException if a {@link Coder} cannot be provided
    */
-  public <InputT, OutputT> Coder<OutputT> getDefaultCoder(
+  @Deprecated
+  @Internal
+  public <InputT, OutputT> Coder<OutputT> getCoder(
       TypeDescriptor<OutputT> typeDescriptor,
       TypeDescriptor<InputT> inputTypeDescriptor,
       Coder<InputT> inputCoder)
@@ -258,22 +245,26 @@ public <InputT, OutputT> Coder<OutputT> getDefaultCoder(
     checkArgument(typeDescriptor != null);
     checkArgument(inputTypeDescriptor != null);
     checkArgument(inputCoder != null);
-    return getDefaultCoder(
+    return getCoderFromTypeDescriptor(
         typeDescriptor, getTypeToCoderBindings(inputTypeDescriptor.getType(), inputCoder));
   }
 
   /**
    * Returns the {@link Coder} to use on elements produced by this function, given the {@link Coder}
    * used for its input elements.
+   *
+   * @throws CannotProvideCoderException if a {@link Coder} cannot be provided
    */
-  public <InputT, OutputT> Coder<OutputT> getDefaultOutputCoder(
+  @Deprecated
+  @Internal
+  public <InputT, OutputT> Coder<OutputT> getOutputCoder(
       SerializableFunction<InputT, OutputT> fn, Coder<InputT> inputCoder)
       throws CannotProvideCoderException {
 
     ParameterizedType fnType = (ParameterizedType)
         TypeDescriptor.of(fn.getClass()).getSupertype(SerializableFunction.class).getType();
 
-    return getDefaultCoder(
+    return getCoder(
         fn.getClass(),
         SerializableFunction.class,
         ImmutableMap.of(fnType.getActualTypeArguments()[0], inputCoder),
@@ -284,9 +275,11 @@ public <InputT, OutputT> Coder<OutputT> getDefaultOutputCoder(
    * Returns the {@link Coder} to use for the specified type parameter specialization of the
    * subclass, given {@link Coder Coders} to use for all other type parameters (if any).
    *
-   * @throws CannotProvideCoderException if there is no default Coder.
+   * @throws CannotProvideCoderException if a {@link Coder} cannot be provided
    */
-  public <T, OutputT> Coder<OutputT> getDefaultCoder(
+  @Deprecated
+  @Internal
+  public <T, OutputT> Coder<OutputT> getCoder(
       Class<? extends T> subClass,
       Class<T> baseClass,
       Map<Type, ? extends Coder<?>> knownCoders,
@@ -305,147 +298,11 @@ public <T, OutputT> Coder<OutputT> getDefaultCoder(
     }
   }
 
-  /**
-   * Returns the {@link Coder} to use for the provided example value, if it can be determined.
-   *
-   * @throws CannotProvideCoderException if there is no default {@link Coder} or
-   * more than one {@link Coder} matches
-   */
-  public <T> Coder<T> getDefaultCoder(T exampleValue) throws CannotProvideCoderException {
-    Class<?> clazz = exampleValue == null ? Void.class : exampleValue.getClass();
-
-    if (clazz.getTypeParameters().length == 0) {
-      // Trust that getDefaultCoder returns a valid
-      // Coder<T> for non-generic clazz.
-      @SuppressWarnings("unchecked")
-      Coder<T> coder = (Coder<T>) getDefaultCoder(clazz);
-      return coder;
-    } else {
-      CoderFactory factory = getDefaultCoderFactory(clazz);
-
-      List<Object> components = factory.getInstanceComponents(exampleValue);
-      if (components == null) {
-        throw new CannotProvideCoderException(String.format(
-            "Cannot provide coder based on value with class %s: The registered CoderFactory with "
-            + "class %s failed to decompose the value, which is required in order to provide "
-            + "Coders for the components.",
-            clazz.getCanonicalName(), factory.getClass().getCanonicalName()));
-      }
-
-      // componentcoders = components.map(this.getDefaultCoder)
-      List<Coder<?>> componentCoders = new ArrayList<>();
-      for (Object component : components) {
-        try {
-          Coder<?> componentCoder = getDefaultCoder(component);
-          componentCoders.add(componentCoder);
-        } catch (CannotProvideCoderException exc) {
-          throw new CannotProvideCoderException(
-              String.format("Cannot provide coder based on value with class %s",
-                  clazz.getCanonicalName()),
-              exc);
-        }
-      }
-
-      // Trust that factory.create maps from valid component Coders
-      // to a valid Coder<T>.
-      @SuppressWarnings("unchecked")
-      Coder<T> coder = (Coder<T>) factory.create(componentCoders);
-      return coder;
-    }
-  }
-
-  /**
-   * Returns the {@link Coder} to use by default for values of the given class. The following three
-   * sources for a {@link Coder} will be attempted, in order:
-   *
-   * <ol>
-   *   <li>A {@link Coder} class registered explicitly via a call to {@link #registerCoder},
-   *   <li>A {@link DefaultCoder} annotation on the class,
-   *   <li>This registry's fallback {@link CoderProvider}, which may be able to generate a
-   *   {@link Coder} for an arbitrary class.
-   * </ol>
-   *
-   * @throws CannotProvideCoderException if a {@link Coder} cannot be provided
-   */
-  public <T> Coder<T> getDefaultCoder(Class<T> clazz) throws CannotProvideCoderException {
-
-    CannotProvideCoderException factoryException;
-    try {
-      CoderFactory coderFactory = getDefaultCoderFactory(clazz);
-      LOG.debug("Default coder for {} found by factory", clazz);
-      @SuppressWarnings("unchecked")
-      Coder<T> coder = (Coder<T>) coderFactory.create(Collections.<Coder<?>>emptyList());
-      return coder;
-    } catch (CannotProvideCoderException exc) {
-      factoryException = exc;
-    }
-
-    CannotProvideCoderException annotationException;
-    try {
-      return getDefaultCoderFromAnnotation(clazz);
-    } catch (CannotProvideCoderException exc) {
-      annotationException = exc;
-    }
-
-    CannotProvideCoderException fallbackException;
-    if (getFallbackCoderProvider() != null) {
-      try {
-        return getFallbackCoderProvider().getCoder(TypeDescriptor.<T>of(clazz));
-      } catch (CannotProvideCoderException exc) {
-        fallbackException = exc;
-      }
-    } else {
-      fallbackException = new CannotProvideCoderException("no fallback CoderProvider configured");
-    }
-
-    // Build up the error message and list of causes.
-    StringBuilder messageBuilder = new StringBuilder()
-        .append("Unable to provide a default Coder for ").append(clazz.getCanonicalName())
-        .append(". Correct one of the following root causes:");
-
-    messageBuilder
-        .append("\n  Building a Coder using a registered CoderFactory failed: ")
-        .append(factoryException.getMessage());
-
-    messageBuilder
-        .append("\n  Building a Coder from the @DefaultCoder annotation failed: ")
-        .append(annotationException.getMessage());
-
-    messageBuilder
-        .append("\n  Building a Coder from the fallback CoderProvider failed: ")
-        .append(fallbackException.getMessage());
-
-    throw new CannotProvideCoderException(messageBuilder.toString());
-  }
-
-  /**
-   * Sets the fallback {@link CoderProvider} for this registry. If no other method succeeds in
-   * providing a {@code Coder<T>} for a type {@code T}, then the registry will attempt to create
-   * a {@link Coder} using this {@link CoderProvider}.
-   *
-   * <p>By default, this is set to {@link SerializableCoder#PROVIDER}.
-   *
-   * <p>See {@link #getFallbackCoderProvider}.
-   */
-  public void setFallbackCoderProvider(CoderProvider coderProvider) {
-    fallbackCoderProvider = coderProvider;
-  }
-
-  /**
-   * Returns the fallback {@link CoderProvider} for this registry.
-   *
-   * <p>See {@link #setFallbackCoderProvider}.
-   */
-  public CoderProvider getFallbackCoderProvider() {
-    return fallbackCoderProvider;
-  }
-
   /////////////////////////////////////////////////////////////////////////////
 
   /**
    * Returns a {@code Map} from each of {@code baseClass}'s type parameters to the {@link Coder} to
-   * use by default for it, in the context of {@code subClass}'s specialization of
-   * {@code baseClass}.
+   * use for it, in the context of {@code subClass}'s specialization of {@code baseClass}.
    *
    * <p>If no {@link Coder} can be inferred for a particular type parameter, then that type variable
    * will be absent from the returned {@code Map}.
@@ -465,7 +322,7 @@ public CoderProvider getFallbackCoderProvider() {
    * <p>For this reason {@code getDefaultCoders} (plural) does not throw an exception if a
    * {@link Coder} for a particular type variable cannot be inferred, but merely omits the entry
    * from the returned {@code Map}. It is the responsibility of the caller (usually
-   * {@link #getDefaultCoder} to extract the desired coder or throw a
+   * {@link #getCoderFromTypeDescriptor} to extract the desired coder or throw a
    * {@link CannotProvideCoderException} when appropriate.
    *
    * @param subClass the concrete type whose specializations are being inferred
@@ -495,8 +352,7 @@ private <T> Map<Type, Coder<?>> getDefaultCoders(
 
   /**
    * Returns an array listing, for each of {@code baseClass}'s type parameters, the {@link Coder} to
-   * use by default for it, in the context of {@code subClass}'s specialization of
-   * {@code baseClass}.
+   * use for it, in the context of {@code subClass}'s specialization of {@code baseClass}.
    *
    * <p>If a {@link Coder} cannot be inferred for a type variable, its slot in the resulting array
    * will be {@code null}.
@@ -518,7 +374,7 @@ private <T> Map<Type, Coder<?>> getDefaultCoders(
    * <p>For this reason {@code getDefaultCoders} (plural) does not throw an exception if a
    * {@link Coder} for a particular type variable cannot be inferred. Instead, it results in a
    * {@code null} in the array. It is the responsibility of the caller (usually
-   * {@link #getDefaultCoder} to extract the desired coder or throw a
+   * {@link #getCoderFromTypeDescriptor} to extract the desired coder or throw a
    * {@link CannotProvideCoderException} when appropriate.
    *
    * @param subClass the concrete type whose specializations are being inferred
@@ -571,7 +427,7 @@ private <T> Coder<?>[] getDefaultCoders(
         result[i] = knownCoders[i];
       } else {
         try {
-          result[i] = getDefaultCoder(typeArgs[i], context);
+          result[i] = getCoderFromTypeDescriptor(TypeDescriptor.of(typeArgs[i]), context);
         } catch (CannotProvideCoderException exc) {
           result[i] = null;
         }
@@ -695,89 +551,31 @@ private static boolean isNullOrEmpty(Collection<?> c) {
   }
 
   /**
-   * The map of classes to the CoderFactories to use to create their
-   * default Coders.
+   * The list of {@link CoderProvider coder providers} to use to provide Coders.
    */
-  private Map<Class<?>, CoderFactory> coderFactoryMap;
-
-  /**
-   * A provider of coders for types where no coder is registered.
-   */
-  private CoderProvider fallbackCoderProvider;
-
-  /**
-   * Returns the {@link CoderFactory} to use to create default {@link Coder Coders} for instances of
-   * the given class, or {@code null} if there is no default {@link CoderFactory} registered.
-   */
-  private CoderFactory getDefaultCoderFactory(Class<?> clazz) throws CannotProvideCoderException {
-    CoderFactory coderFactoryOrNull = coderFactoryMap.get(clazz);
-    if (coderFactoryOrNull != null) {
-      return coderFactoryOrNull;
-    } else {
-      throw new CannotProvideCoderException(
-          String.format("Cannot provide coder based on value with class %s: No CoderFactory has "
-              + "been registered for the class.", clazz.getCanonicalName()));
-    }
-  }
-
-  /**
-   * Returns the {@link Coder} returned according to the {@link CoderProvider} from any
-   * {@link DefaultCoder} annotation on the given class.
-   */
-  private <T> Coder<T> getDefaultCoderFromAnnotation(Class<T> clazz)
-      throws CannotProvideCoderException {
-    DefaultCoder defaultAnnotation = clazz.getAnnotation(DefaultCoder.class);
-    if (defaultAnnotation == null) {
-      throw new CannotProvideCoderException(
-          String.format("Class %s does not have a @DefaultCoder annotation.",
-              clazz.getCanonicalName()));
-    }
-
-    LOG.debug("DefaultCoder annotation found for {} with value {}",
-        clazz, defaultAnnotation.value());
-    CoderProvider coderProvider = CoderProviders.fromStaticMethods(defaultAnnotation.value());
-    return coderProvider.getCoder(TypeDescriptor.of(clazz));
-  }
-
-  /**
-   * Returns the {@link Coder} to use by default for values of the given type,
-   * in a context where the given types use the given coders.
-   *
-   * @throws CannotProvideCoderException if a coder cannot be provided
-   */
-  private <T> Coder<T> getDefaultCoder(
-      TypeDescriptor<T> typeDescriptor,
-      Map<Type, Coder<?>> typeCoderBindings)
-      throws CannotProvideCoderException {
-
-    Coder<?> defaultCoder = getDefaultCoder(typeDescriptor.getType(), typeCoderBindings);
-    LOG.debug("Default coder for {}: {}", typeDescriptor, defaultCoder);
-    @SuppressWarnings("unchecked")
-    Coder<T> result = (Coder<T>) defaultCoder;
-    return result;
-  }
+  private LinkedList<CoderProvider> coderProviders;
 
   /**
-   * Returns the {@link Coder} to use by default for values of the given type,
+   * Returns a {@link Coder} to use for values of the given type,
    * in a context where the given types use the given coders.
    *
    * @throws CannotProvideCoderException if a coder cannot be provided
    */
-  private Coder<?> getDefaultCoder(Type type, Map<Type, Coder<?>> typeCoderBindings)
+  private <T> Coder<T> getCoderFromTypeDescriptor(
+      TypeDescriptor<T> typeDescriptor, Map<Type, Coder<?>> typeCoderBindings)
       throws CannotProvideCoderException {
-    Coder<?> coder = typeCoderBindings.get(type);
-    if (coder != null) {
-      return coder;
-    }
-    if (type instanceof Class<?>) {
-      Class<?> clazz = (Class<?>) type;
-      return getDefaultCoder(clazz);
+    Type type = typeDescriptor.getType();
+    Coder<?> coder;
+    if (typeCoderBindings.containsKey(type)) {
+      coder = typeCoderBindings.get(type);
+    } else if (type instanceof Class<?>) {
+      coder = getCoderFromFactories(typeDescriptor, Collections.<Coder<?>>emptyList());
     } else if (type instanceof ParameterizedType) {
-      return getDefaultCoder((ParameterizedType) type, typeCoderBindings);
+      coder = getCoderFromParameterizedType((ParameterizedType) type, typeCoderBindings);
     } else if (type instanceof TypeVariable) {
-      return getDefaultCoder(TypeDescriptor.of(type).getRawType());
+      coder = getCoderFromFactories(typeDescriptor, Collections.<Coder<?>>emptyList());
     } else if (type instanceof WildcardType) {
-      // No default coder for an unknown generic type.
+      // No coder for an unknown generic type.
       throw new CannotProvideCoderException(
           String.format("Cannot provide a coder for type variable %s"
           + " (declared by %s) because the actual type is unknown due to erasure.",
@@ -788,72 +586,70 @@ private Coder<?> getDefaultCoder(Type type, Map<Type, Coder<?>> typeCoderBinding
       throw new RuntimeException(
           "Internal error: unexpected kind of Type: " + type);
     }
+
+    LOG.debug("Coder for {}: {}", typeDescriptor, coder);
+    @SuppressWarnings("unchecked")
+    Coder<T> result = (Coder<T>) coder;
+    return result;
   }
 
   /**
-   * Returns the {@link Coder} to use by default for values of the given
+   * Returns a {@link Coder} to use for values of the given
    * parameterized type, in a context where the given types use the
    * given {@link Coder Coders}.
    *
    * @throws CannotProvideCoderException if no coder can be provided
    */
-  private Coder<?> getDefaultCoder(
+  private Coder<?> getCoderFromParameterizedType(
       ParameterizedType type,
       Map<Type, Coder<?>> typeCoderBindings)
           throws CannotProvideCoderException {
 
-    CannotProvideCoderException factoryException;
-    try {
-      return getDefaultCoderFromFactory(type, typeCoderBindings);
-    } catch (CannotProvideCoderException exc) {
-      factoryException = exc;
-    }
-
-    CannotProvideCoderException annotationException;
-    try {
-      Class<?> rawClazz = (Class<?>) type.getRawType();
-      return getDefaultCoderFromAnnotation(rawClazz);
-    } catch (CannotProvideCoderException exc) {
-      annotationException = exc;
-    }
-
-    // Build up the error message and list of causes.
-    StringBuilder messageBuilder = new StringBuilder()
-        .append("Unable to provide a default Coder for ").append(type)
-        .append(". Correct one of the following root causes:");
-
-    messageBuilder
-        .append("\n  Building a Coder using a registered CoderFactory failed: ")
-        .append(factoryException.getMessage());
-
-    messageBuilder
-        .append("\n  Building a Coder from the @DefaultCoder annotation failed: ")
-        .append(annotationException.getMessage());
-
-    throw new CannotProvideCoderException(messageBuilder.toString());
-  }
-
-  private Coder<?> getDefaultCoderFromFactory(
-      ParameterizedType type,
-      Map<Type, Coder<?>> typeCoderBindings)
-          throws CannotProvideCoderException {
-    Class<?> rawClazz = (Class<?>) type.getRawType();
-    CoderFactory coderFactory = getDefaultCoderFactory(rawClazz);
     List<Coder<?>> typeArgumentCoders = new ArrayList<>();
     for (Type typeArgument : type.getActualTypeArguments()) {
       try {
-        Coder<?> typeArgumentCoder = getDefaultCoder(typeArgument,
-                                                     typeCoderBindings);
+        Coder<?> typeArgumentCoder =
+            getCoderFromTypeDescriptor(TypeDescriptor.of(typeArgument), typeCoderBindings);
         typeArgumentCoders.add(typeArgumentCoder);
       } catch (CannotProvideCoderException exc) {
-         throw new CannotProvideCoderException(
-          String.format("Cannot provide coder for parameterized type %s: %s",
-              type,
-              exc.getMessage()),
-          exc);
+        throw new CannotProvideCoderException(
+            String.format("Cannot provide coder for parameterized type %s: %s",
+                type,
+                exc.getMessage()),
+            exc);
       }
     }
-    return coderFactory.create(typeArgumentCoders);
+    return getCoderFromFactories(TypeDescriptor.of(type), typeArgumentCoders);
+  }
+
+  /**
+   * Attempts to create a {@link Coder} from any registered {@link CoderProvider} returning
+   * the first successfully created instance.
+   */
+  private Coder<?> getCoderFromFactories(
+      TypeDescriptor<?> typeDescriptor, List<Coder<?>> typeArgumentCoders)
+      throws CannotProvideCoderException {
+    List<CannotProvideCoderException> suppressedExceptions = new ArrayList<>();
+    for (CoderProvider coderProvider : coderProviders) {
+      try {
+        return coderProvider.coderFor(typeDescriptor, typeArgumentCoders);
+      } catch (CannotProvideCoderException e) {
+        // Add all failures as suppressed exceptions.
+        suppressedExceptions.add(e);
+      }
+    }
+
+    // Build up the error message and list of causes.
+    StringBuilder messageBuilder = new StringBuilder()
+        .append("Unable to provide a Coder for ").append(typeDescriptor).append(".\n")
+        .append("  Building a Coder using a registered CoderProvider failed.\n")
+        .append("  See suppressed exceptions for detailed failures.");
+    CannotProvideCoderException exceptionOnFailure =
+        new CannotProvideCoderException(messageBuilder.toString());
+    for (CannotProvideCoderException suppressedException : suppressedExceptions) {
+      exceptionOnFailure.addSuppressed(suppressedException);
+    }
+    throw exceptionOnFailure;
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CollectionCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CollectionCoder.java
index 523b69ba6e19..17623083bb47 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CollectionCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CollectionCoder.java
@@ -46,15 +46,6 @@ protected final Collection<T> decodeToIterable(List<T> decodedElements) {
     return decodedElements;
   }
 
-  /**
-   * Returns the first element in this collection if it is non-empty,
-   * otherwise returns {@code null}.
-   */
-  public static <T> List<Object> getInstanceComponents(
-      Collection<T> exampleValue) {
-    return getInstanceComponentsHelper(exampleValue);
-  }
-
   protected CollectionCoder(Coder<T> elemCoder) {
     super(elemCoder, "Collection");
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
index f33e2104d3d7..edbaa7f92fc8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
@@ -93,14 +93,6 @@ public List<? extends Coder<?>> getCoderArguments() {
     return Collections.emptyList();
   }
 
-  /**
-   * Returns an empty list. A {@link CustomCoder} by default will not have component coders that are
-   * used for inference.
-   */
-  public static <T> List<Object> getInstanceComponents(T exampleValue) {
-    return Collections.emptyList();
-  }
-
   /**
    * {@inheritDoc}
    *
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java
index 9a976f9d4cc4..6eff9e9654b0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java
@@ -17,45 +17,33 @@
  */
 package org.apache.beam.sdk.coders;
 
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
 import java.lang.annotation.Documented;
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
- * The {@link DefaultCoder} annotation
- * specifies a default {@link Coder} class to handle encoding and decoding
- * instances of the annotated class.
+ * The {@link DefaultCoder} annotation specifies a {@link Coder} class to handle encoding and
+ * decoding instances of the annotated class.
  *
- * <p>The specified {@link Coder} must satisfy the requirements of
- * {@link CoderProviders#fromStaticMethods}. Two classes provided by the SDK that
- * are intended for use with this annotation include {@link SerializableCoder}
- * and {@link AvroCoder}.
+ * <p>The specified {@link Coder} must have the following method:
+ * <pre>
+ * {@code public static CoderProvider getCoderProvider()}.
+ * </pre>
  *
- * <p>To configure the use of Java serialization as the default
- * for a class, annotate the class to use
- * {@link SerializableCoder} as follows:
- *
- * <pre><code>{@literal @}DefaultCoder(SerializableCoder.class)
- * public class MyCustomDataType implements Serializable {
- *   // ...
- * }</code></pre>
- *
- * <p>Similarly, to configure the use of
- * {@link AvroCoder} as the default:
- * <pre><code>{@literal @}DefaultCoder(AvroCoder.class)
- * public class MyCustomDataType {
- *   public MyCustomDataType() {}  // Avro requires an empty constructor.
- *   // ...
- * }</code></pre>
- *
- * <p>Coders specified explicitly via
- * {@link PCollection#setCoder}
- * take precedence, followed by Coders registered at runtime via
- * {@link CoderRegistry#registerCoder}. See {@link CoderRegistry} for a more detailed discussion
- * of the precedence rules.
+ * <p>Coders specified explicitly via {@link PCollection#setCoder} take precedence, followed by
+ * Coders found at runtime via {@link CoderRegistry#getCoder}.
+ * See {@link CoderRegistry} for a more detailed discussion of the precedence rules.
  */
 @Documented
 @Retention(RetentionPolicy.RUNTIME)
@@ -63,4 +51,77 @@
 @SuppressWarnings("rawtypes")
 public @interface DefaultCoder {
   Class<? extends Coder> value();
+
+  /**
+   * A {@link CoderProviderRegistrar} that registers a {@link CoderProvider} which can use
+   * the {@code @DefaultCoder} annotation to provide {@link CoderProvider coder providers} that
+   * creates {@link Coder}s.
+   */
+  @AutoService(CoderProviderRegistrar.class)
+  class DefaultCoderProviderRegistrar implements CoderProviderRegistrar {
+
+    @Override
+    public List<CoderProvider> getCoderProviders() {
+      return ImmutableList.<CoderProvider>of(new DefaultCoderProvider());
+    }
+
+    /**
+     * A {@link CoderProvider} that uses the {@code @DefaultCoder} annotation to provide
+     * {@link CoderProvider coder providers} that create {@link Coder}s.
+     */
+    static class DefaultCoderProvider extends CoderProvider {
+      private static final Logger LOG = LoggerFactory.getLogger(DefaultCoderProvider.class);
+
+      /**
+       * Returns the {@link Coder} returned according to the {@link CoderProvider} from any
+       * {@link DefaultCoder} annotation on the given class.
+       */
+      @Override
+      public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+          List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+
+        Class<?> clazz = typeDescriptor.getRawType();
+        DefaultCoder defaultAnnotation = clazz.getAnnotation(DefaultCoder.class);
+        if (defaultAnnotation == null) {
+          throw new CannotProvideCoderException(
+              String.format("Class %s does not have a @DefaultCoder annotation.",
+                  clazz.getName()));
+        }
+
+        if (defaultAnnotation.value() == null) {
+          throw new CannotProvideCoderException(
+              String.format("Class %s has a @DefaultCoder annotation with a null value.",
+                  clazz.getName()));
+        }
+
+        LOG.debug("DefaultCoder annotation found for {} with value {}",
+            clazz, defaultAnnotation.value());
+
+        Method coderProviderMethod;
+        try {
+          coderProviderMethod = defaultAnnotation.value().getMethod("getCoderProvider");
+        } catch (NoSuchMethodException e) {
+          throw new CannotProvideCoderException(String.format(
+              "Unable to find 'public static CoderProvider getCoderProvider()' on %s",
+              defaultAnnotation.value()),
+              e);
+        }
+
+        CoderProvider coderProvider;
+        try {
+          coderProvider = (CoderProvider) coderProviderMethod.invoke(null);
+        } catch (IllegalAccessException
+            | IllegalArgumentException
+            | InvocationTargetException
+            | NullPointerException
+            | ExceptionInInitializerError e) {
+          throw new CannotProvideCoderException(String.format(
+              "Unable to invoke 'public static CoderProvider getCoderProvider()' on %s",
+              defaultAnnotation.value()),
+              e);
+        }
+        return coderProvider.coderFor(typeDescriptor, componentCoders);
+      }
+    }
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableCoder.java
index 02c3d0fbebca..b600b1d5d107 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableCoder.java
@@ -41,15 +41,6 @@ protected final Iterable<T> decodeToIterable(List<T> decodedElements) {
     return decodedElements;
   }
 
-  /**
-   * Returns the first element in this iterable if it is non-empty,
-   * otherwise returns {@code null}.
-   */
-  public static <T> List<Object> getInstanceComponents(
-      Iterable<T> exampleValue) {
-    return getInstanceComponentsHelper(exampleValue);
-  }
-
   protected IterableCoder(Coder<T> elemCoder) {
     super(elemCoder, "Iterable");
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
index 8e10ca25cdd7..52b9c34c3c92 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
@@ -75,18 +75,6 @@ public Coder<T> getElemCoder() {
   private final Coder<T> elementCoder;
   private final String iterableName;
 
-  /**
-   * Returns the first element in the iterable-like {@code exampleValue} if it is non-empty,
-   * otherwise returns {@code null}.
-   */
-  protected static <T, IterableT extends Iterable<T>>
-      List<Object> getInstanceComponentsHelper(IterableT exampleValue) {
-    for (T value : exampleValue) {
-      return Arrays.<Object>asList(value);
-    }
-    return null;
-  }
-
   protected IterableLikeCoder(Coder<T> elementCoder, String  iterableName) {
     checkArgument(elementCoder != null, "element Coder for IterableLikeCoder must not be null");
     checkArgument(iterableName != null, "iterable name for IterableLikeCoder must not be null");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
index 35b7449bcb43..da7f03c7599e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
@@ -39,13 +39,6 @@ public static <K, V> KvCoder<K, V> of(Coder<K> keyCoder,
     return new KvCoder<>(keyCoder, valueCoder);
   }
 
-  public static <K, V> List<Object> getInstanceComponents(
-      KV<K, V> exampleValue) {
-    return Arrays.asList(
-        exampleValue.getKey(),
-        exampleValue.getValue());
-  }
-
   public Coder<K> getKeyCoder() {
     return keyCoder;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java
index 70bbf9333787..25f3ee9b1479 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ListCoder.java
@@ -40,14 +40,6 @@ protected final List<T> decodeToIterable(List<T> decodedElements) {
     return decodedElements;
   }
 
-  /**
-   * Returns the first element in this list if it is non-empty,
-   * otherwise returns {@code null}.
-   */
-  public static <T> List<Object> getInstanceComponents(List<T> exampleValue) {
-    return getInstanceComponentsHelper(exampleValue);
-  }
-
   protected ListCoder(Coder<T> elemCoder) {
     super(elemCoder, "List");
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
index da2bf50d3125..9e3c76829f88 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
@@ -50,18 +50,6 @@ public static <K, V> MapCoder<K, V> of(
     return new MapCoder<>(keyCoder, valueCoder);
   }
 
-  /**
-   * Returns the key and value for an arbitrary element of this map,
-   * if it is non-empty, otherwise returns {@code null}.
-   */
-   public static <K, V> List<Object> getInstanceComponents(
-       Map<K, V> exampleValue) {
-     for (Map.Entry<K, V> entry : exampleValue.entrySet()) {
-       return Arrays.asList(entry.getKey(), entry.getValue());
-     }
-     return null;
-   }
-
   public Coder<K> getKeyCoder() {
     return keyCoder;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
index b52b9dba114b..e3b2959fd98b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
@@ -17,12 +17,15 @@
  */
 package org.apache.beam.sdk.coders;
 
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.util.List;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
@@ -63,29 +66,45 @@ public static <T extends Serializable> SerializableCoder<T> of(Class<T> clazz) {
   }
 
   /**
-   * A {@link CoderProvider} that constructs a {@link SerializableCoder}
-   * for any class that implements serializable.
+   * Returns a {@link CoderProvider} which uses the {@link SerializableCoder} if possible for
+   * all types.
+   *
+   * <p>This method is invoked reflectively from {@link DefaultCoder}.
    */
-  public static final CoderProvider PROVIDER = new CoderProvider() {
+  @SuppressWarnings("unused")
+  public static CoderProvider getCoderProvider() {
+    return new SerializableCoderProvider();
+  }
+
+  /**
+   * A {@link CoderProviderRegistrar} which registers a {@link CoderProvider} which can handle
+   * serializable types.
+   */
+  @AutoService(CoderProviderRegistrar.class)
+  public static class SerializableCoderProviderRegistrar implements CoderProviderRegistrar {
+
     @Override
-    public <T> Coder<T> getCoder(TypeDescriptor<T> typeDescriptor)
-        throws CannotProvideCoderException {
-      Class<?> clazz = typeDescriptor.getRawType();
-      if (Serializable.class.isAssignableFrom(clazz)) {
-        @SuppressWarnings("unchecked")
-        Class<? extends Serializable> serializableClazz =
-            (Class<? extends Serializable>) clazz;
-        @SuppressWarnings("unchecked")
-        Coder<T> coder = (Coder<T>) SerializableCoder.of(serializableClazz);
-        return coder;
-      } else {
-        throw new CannotProvideCoderException(
-            "Cannot provide SerializableCoder because " + typeDescriptor
-            + " does not implement Serializable");
-      }
+    public List<CoderProvider> getCoderProviders() {
+      return ImmutableList.of(getCoderProvider());
     }
-  };
+  }
 
+  /**
+   * A {@link CoderProvider} that constructs a {@link SerializableCoder} for any class that
+   * implements serializable.
+   */
+  static class SerializableCoderProvider extends CoderProvider {
+    @Override
+    public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+        List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+      if (Serializable.class.isAssignableFrom(typeDescriptor.getRawType())) {
+        return SerializableCoder.of((TypeDescriptor) typeDescriptor);
+      }
+      throw new CannotProvideCoderException(
+          "Cannot provide SerializableCoder because " + typeDescriptor
+              + " does not implement Serializable");
+    }
+  }
 
   private final Class<T> type;
   private final TypeDescriptor<T> typeDescriptor;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SetCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SetCoder.java
index da161659181e..baec12807623 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SetCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SetCoder.java
@@ -56,15 +56,6 @@ public TypeDescriptor<Set<T>> getEncodedTypeDescriptor() {
         new TypeParameter<T>() {}, getElemCoder().getEncodedTypeDescriptor());
   }
 
-  /**
-   * Returns the first element in this set if it is non-empty,
-   * otherwise returns {@code null}.
-   */
-  public static <T> List<Object> getInstanceComponents(
-      Set<T> exampleValue) {
-    return getInstanceComponentsHelper(exampleValue);
-  }
-
   /////////////////////////////////////////////////////////////////////////////
   // Internal operations below here.
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
index 7fc094f755b9..9a7b125cbca1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
@@ -37,13 +37,6 @@ public static VarLongCoder of() {
     return INSTANCE;
   }
 
-  /**
-   * Returns an empty list. {@link VarLongCoder} has no components.
-   */
-  public static <T> List<Object> getInstanceComponents(T ignored) {
-    return Collections.emptyList();
-  }
-
   /////////////////////////////////////////////////////////////////////////////
 
   private static final VarLongCoder INSTANCE = new VarLongCoder();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
index 29990cd60b00..20ec3003f2a1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java
@@ -125,14 +125,14 @@ abstract static class AbstractGlobalCombineFn<InputT, AccumT, OutputT>
     @Override
     public Coder<AccumT> getAccumulatorCoder(CoderRegistry registry, Coder<InputT> inputCoder)
         throws CannotProvideCoderException {
-      return registry.getDefaultCoder(getClass(), AbstractGlobalCombineFn.class,
+      return registry.getCoder(getClass(), AbstractGlobalCombineFn.class,
           ImmutableMap.<Type, Coder<?>>of(getInputTVariable(), inputCoder), getAccumTVariable());
     }
 
     @Override
     public Coder<OutputT> getDefaultOutputCoder(CoderRegistry registry, Coder<InputT> inputCoder)
         throws CannotProvideCoderException {
-      return registry.getDefaultCoder(getClass(), AbstractGlobalCombineFn.class,
+      return registry.getCoder(getClass(), AbstractGlobalCombineFn.class,
           ImmutableMap.<Type, Coder<?>>of(getInputTVariable(), inputCoder, getAccumTVariable(),
               this.getAccumulatorCoder(registry, inputCoder)),
           getOutputTVariable());
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
index d4c97bc3c4c9..0515ed527e50 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
@@ -339,7 +339,7 @@ public Coder<Object[]> getAccumulatorCoder(CoderRegistry registry, Coder<DataT>
       List<Coder<Object>> coders = Lists.newArrayList();
       for (int i = 0; i < combineFnCount; ++i) {
         Coder<Object> inputCoder =
-            registry.getDefaultOutputCoder(extractInputFns.get(i), dataCoder);
+            registry.getOutputCoder(extractInputFns.get(i), dataCoder);
         coders.add(combineFns.get(i).getAccumulatorCoder(registry, inputCoder));
       }
       return new ComposedAccumulatorCoder(coders);
@@ -478,7 +478,7 @@ public Coder<Object[]> getAccumulatorCoder(CoderRegistry registry, Coder<DataT>
       List<Coder<Object>> coders = Lists.newArrayList();
       for (int i = 0; i < combineFnCount; ++i) {
         Coder<Object> inputCoder =
-            registry.getDefaultOutputCoder(extractInputFns.get(i), dataCoder);
+            registry.getOutputCoder(extractInputFns.get(i), dataCoder);
         coders.add(combineFnWithContexts.get(i).getAccumulatorCoder(registry, inputCoder));
       }
       return new ComposedAccumulatorCoder(coders);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
index 5624f2f2cbe4..7af8fb8fbaa4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
@@ -27,17 +27,25 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.Set;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.CollectionCoder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.coders.MapCoder;
+import org.apache.beam.sdk.coders.SetCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.OffsetBasedSource;
@@ -317,7 +325,7 @@ public Coder<T> getDefaultOutputCoder(PBegin input) throws CannotProvideCoderExc
       if (coder.isPresent()) {
         return coder.get();
       } else if (typeDescriptor.isPresent()) {
-        return input.getPipeline().getCoderRegistry().getDefaultCoder(typeDescriptor.get());
+        return input.getPipeline().getCoderRegistry().getCoder(typeDescriptor.get());
       } else {
         return getDefaultCreateCoder(input.getPipeline().getCoderRegistry(), elems);
       }
@@ -566,7 +574,7 @@ public Coder<T> getDefaultOutputCoder(PBegin input) throws CannotProvideCoderExc
       if (elementCoder.isPresent()) {
         return elementCoder.get();
       } else if (typeDescriptor.isPresent()) {
-        return input.getPipeline().getCoderRegistry().getDefaultCoder(typeDescriptor.get());
+        return input.getPipeline().getCoderRegistry().getCoder(typeDescriptor.get());
       } else {
         Iterable<T> rawElements =
             Iterables.transform(
@@ -611,7 +619,7 @@ private static <T> Coder<T> getDefaultCreateCoder(CoderRegistry registry, Iterab
     if (elementClazz.getTypeParameters().length == 0) {
       try {
         @SuppressWarnings("unchecked") // elementClazz is a wildcard type
-        Coder<T> coder = (Coder<T>) registry.getDefaultCoder(TypeDescriptor.of(elementClazz));
+        Coder<T> coder = (Coder<T>) registry.getCoder(TypeDescriptor.of(elementClazz));
         return coder;
       } catch (CannotProvideCoderException exc) {
         // Can't get a coder from the class of the elements, try with the elements next
@@ -619,11 +627,20 @@ private static <T> Coder<T> getDefaultCreateCoder(CoderRegistry registry, Iterab
     }
 
     // If that fails, try to deduce a coder using the elements themselves
-    Optional<Coder<T>> coder = Optional.absent();
-    for (T elem : elems) {
-      Coder<T> c = registry.getDefaultCoder(elem);
+    return (Coder<T>) inferCoderFromObjects(registry, elems);
+  }
+
+  /**
+   * Attempts to infer the {@link Coder} of the elements ensuring that the returned coder is
+   * equivalent for all elements.
+   */
+  private static Coder<?> inferCoderFromObjects(
+      CoderRegistry registry, Iterable<?> elems) throws CannotProvideCoderException {
+    Optional<Coder<?>> coder = Optional.absent();
+    for (Object elem : elems) {
+      Coder<?> c = inferCoderFromObject(registry, elem);
       if (!coder.isPresent()) {
-        coder = Optional.of(c);
+        coder = (Optional) Optional.of(c);
       } else if (!Objects.equals(c, coder.get())) {
         throw new CannotProvideCoderException(
             "Cannot provide coder for elements of "
@@ -633,11 +650,48 @@ private static <T> Coder<T> getDefaultCreateCoder(CoderRegistry registry, Iterab
                 + " Based on their values, they do not all default to the same Coder.");
       }
     }
+    if (coder.isPresent()) {
+      return coder.get();
+    }
 
-    if (!coder.isPresent()) {
-      throw new CannotProvideCoderException(
-          "Unable to infer a coder. Please register " + "a coder for ");
+    throw new CannotProvideCoderException("Cannot provide coder for elements of "
+        + Create.class.getSimpleName()
+        + ":"
+        + " For their common class, no coder could be provided."
+        + " Based on their values, no coder could be inferred.");
+  }
+
+  /**
+   * Attempt to infer the type for some very common Apache Beam parameterized types.
+   *
+   * <p>TODO: Instead, build a TypeDescriptor so that the {@link CoderRegistry} is invoked
+   * for the type instead of hard coding the coders for common types.
+   */
+  private static Coder<?> inferCoderFromObject(CoderRegistry registry, Object o)
+      throws CannotProvideCoderException {
+    if (o == null) {
+      return VoidCoder.of();
+    } else if (o instanceof TimestampedValue) {
+      return TimestampedValueCoder.of(
+          inferCoderFromObject(registry, ((TimestampedValue) o).getValue()));
+    } else if (o instanceof List) {
+      return ListCoder.of(inferCoderFromObjects(registry, ((Iterable) o)));
+    } else if (o instanceof Set) {
+      return SetCoder.of(inferCoderFromObjects(registry, ((Iterable) o)));
+    } else if (o instanceof Collection) {
+      return CollectionCoder.of(inferCoderFromObjects(registry, ((Iterable) o)));
+    } else if (o instanceof Iterable) {
+      return IterableCoder.of(inferCoderFromObjects(registry, ((Iterable) o)));
+    } else if (o instanceof Map) {
+      return MapCoder.of(
+          inferCoderFromObjects(registry, ((Map) o).keySet()),
+          inferCoderFromObjects(registry, ((Map) o).entrySet()));
+    } else if (o instanceof KV) {
+      return KvCoder.of(
+          inferCoderFromObject(registry, ((KV) o).getKey()),
+          inferCoderFromObject(registry, ((KV) o).getValue()));
+    } else {
+      return registry.getCoder(o.getClass());
     }
-    return coder.get();
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
index 15abd9834e75..d5df944330b6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
@@ -117,7 +117,7 @@
  * mapping from Java types to the default Coder to use, for a standard
  * set of Java types; users can extend this mapping for additional
  * types, via
- * {@link org.apache.beam.sdk.coders.CoderRegistry#registerCoder}.
+ * {@link org.apache.beam.sdk.coders.CoderRegistry#registerCoderProvider}.
  * If this inference process fails, either because the Java type was
  * not known at run-time (e.g., due to Java's "erasure" of generic
  * types) or there was no default Coder registered, then the Coder
@@ -281,7 +281,7 @@ private void readObject(ObjectInputStream oos) {
    * @throws CannotProvideCoderException if no coder can be inferred
    */
   protected Coder<?> getDefaultOutputCoder() throws CannotProvideCoderException {
-    throw new CannotProvideCoderException("PTransform.getDefaultOutputCoder called.");
+    throw new CannotProvideCoderException("PTransform.getOutputCoder called.");
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index e67dbe153a5e..edf14191cd1c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -477,10 +477,10 @@ private static <InputT> Coder[] codersForStateSpecTypes(
       Type typeArgument = typeArguments[i];
       TypeDescriptor<?> typeDescriptor = TypeDescriptor.of(typeArgument);
       try {
-        coders[i] = coderRegistry.getDefaultCoder(typeDescriptor);
+        coders[i] = coderRegistry.getCoder(typeDescriptor);
       } catch (CannotProvideCoderException e) {
         try {
-          coders[i] = coderRegistry.getDefaultCoder(
+          coders[i] = coderRegistry.getCoder(
               typeDescriptor, inputCoder.getEncodedTypeDescriptor(), inputCoder);
         } catch (CannotProvideCoderException ignored) {
           // Since not all type arguments will have a registered coder we ignore this exception.
@@ -623,7 +623,7 @@ public PCollection<OutputT> expand(PCollection<? extends InputT> input) {
     @SuppressWarnings("unchecked")
     protected Coder<OutputT> getDefaultOutputCoder(PCollection<? extends InputT> input)
         throws CannotProvideCoderException {
-      return input.getPipeline().getCoderRegistry().getDefaultCoder(
+      return input.getPipeline().getCoderRegistry().getCoder(
           getFn().getOutputTypeDescriptor(),
           getFn().getInputTypeDescriptor(),
           ((PCollection<InputT>) input).getCoder());
@@ -767,7 +767,7 @@ public <T> Coder<T> getDefaultOutputCoder(
         throws CannotProvideCoderException {
       @SuppressWarnings("unchecked")
       Coder<InputT> inputCoder = ((PCollection<InputT>) input).getCoder();
-      return input.getPipeline().getCoderRegistry().getDefaultCoder(
+      return input.getPipeline().getCoderRegistry().getCoder(
           output.getTypeDescriptor(),
           getFn().getInputTypeDescriptor(),
           inputCoder);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java
index dd38006459d8..c66d1b1fb4b8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java
@@ -124,9 +124,9 @@ public KV<K, V> apply(V element) {
       Coder<K> keyCoder;
       CoderRegistry coderRegistry = in.getPipeline().getCoderRegistry();
       if (keyClass == null) {
-        keyCoder = coderRegistry.getDefaultOutputCoder(fn, in.getCoder());
+        keyCoder = coderRegistry.getOutputCoder(fn, in.getCoder());
       } else {
-        keyCoder = coderRegistry.getDefaultCoder(TypeDescriptor.of(keyClass));
+        keyCoder = coderRegistry.getCoder(TypeDescriptor.of(keyClass));
       }
       // TODO: Remove when we can set the coder inference context.
       result.setCoder(KvCoder.of(keyCoder, in.getCoder()));
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
index 0276ba621f59..0bfb8750dcdc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
@@ -87,12 +87,6 @@ public final List<org.apache.beam.sdk.coders.Coder<?>> getCoderArguments() {
       return Collections.emptyList();
     }
 
-    /**
-     * Returns an empty list. The Global Window Coder has no components.
-     */
-    public static <T> List<Object> getInstanceComponents(T exampleValue) {
-      return Collections.emptyList();
-    }
     private Coder() {}
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
index fd2a2d85594f..46ece09e0645 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
@@ -178,13 +178,6 @@ public static IntervalWindowCoder of() {
       return INSTANCE;
     }
 
-    /**
-     * Returns an empty list. {@link IntervalWindowCoder} has no components.
-     */
-    public static <T> List<Object> getInstanceComponents(T value) {
-      return Collections.emptyList();
-    }
-
     @Override
     public void encode(IntervalWindow window, OutputStream outStream, Context context)
         throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
index 1095fb8c7558..f210fd8614e4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
@@ -142,7 +142,7 @@ private CoderOrFailure<T> inferCoderOrFail(
     CannotProvideCoderException inferFromTokenException = null;
     if (token != null) {
       try {
-        return new CoderOrFailure<>(registry.getDefaultCoder(token), null);
+        return new CoderOrFailure<>(registry.getCoder(token), null);
       } catch (CannotProvideCoderException exc) {
         inferFromTokenException = exc;
         // Attempt to detect when the token came from a TupleTag used for a ParDo output,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
index a9f39298b123..c1728857dfbb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
@@ -132,10 +132,6 @@ public Coder<T> getValueCoder() {
       return valueCoder;
     }
 
-    public static <T> List<Object> getInstanceComponents(TimestampedValue<T> exampleValue) {
-      return Arrays.<Object>asList(exampleValue.getValue());
-    }
-
     @Override
     public TypeDescriptor<TimestampedValue<T>> getEncodedTypeDescriptor() {
       return new TypeDescriptor<TimestampedValue<T>>() {}.where(
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderFactoriesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderFactoriesTest.java
deleted file mode 100644
index 4ffc9c1a1984..000000000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderFactoriesTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.coders;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.Collections;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link CoderFactories}.
- */
-@RunWith(JUnit4.class)
-public class CoderFactoriesTest {
-
-  /**
-   * Ensures that a few of our standard atomic coder classes
-   * can each be built into a factory that works as expected.
-   * It is presumed that testing a few, not all, suffices to
-   * exercise CoderFactoryFromStaticMethods.
-   */
-  @Test
-  public void testAtomicCoderClassFactories() {
-    checkAtomicCoderFactory(StringUtf8Coder.class, StringUtf8Coder.of());
-    checkAtomicCoderFactory(DoubleCoder.class, DoubleCoder.of());
-    checkAtomicCoderFactory(ByteArrayCoder.class, ByteArrayCoder.of());
-  }
-
-  /**
-   * Checks that {#link CoderFactories.fromStaticMethods} successfully
-   * builds a working {@link CoderFactory} from {@link KvCoder KvCoder.class}.
-   */
-  @Test
-  public void testKvCoderFactory() {
-    CoderFactory kvCoderFactory = CoderFactories.fromStaticMethods(KvCoder.class);
-    assertEquals(
-        KvCoder.of(DoubleCoder.of(), DoubleCoder.of()),
-        kvCoderFactory.create(Arrays.asList(DoubleCoder.of(), DoubleCoder.of())));
-  }
-
-  /**
-   * Checks that {#link CoderFactories.fromStaticMethods} successfully
-   * builds a working {@link CoderFactory} from {@link ListCoder ListCoder.class}.
-   */
-  @Test
-  public void testListCoderFactory() {
-    CoderFactory listCoderFactory = CoderFactories.fromStaticMethods(ListCoder.class);
-
-    assertEquals(
-        ListCoder.of(DoubleCoder.of()),
-        listCoderFactory.create(Arrays.asList(DoubleCoder.of())));
-  }
-
-  /**
-   * Checks that {#link CoderFactories.fromStaticMethods} successfully
-   * builds a working {@link CoderFactory} from {@link IterableCoder IterableCoder.class}.
-   */
-  @Test
-  public void testIterableCoderFactory() {
-    CoderFactory iterableCoderFactory = CoderFactories.fromStaticMethods(IterableCoder.class);
-
-    assertEquals(
-        IterableCoder.of(DoubleCoder.of()),
-        iterableCoderFactory.create(Arrays.asList(DoubleCoder.of())));
-  }
-
-  ///////////////////////////////////////////////////////////////////////
-
-  /**
-   * Checks that an atomic coder class can be converted into
-   * a factory that then yields a coder equal to the example
-   * provided.
-   */
-  private <T> void checkAtomicCoderFactory(
-      Class<? extends Coder<T>> coderClazz,
-      Coder<T> expectedCoder) {
-    CoderFactory factory = CoderFactories.fromStaticMethods(coderClazz);
-    @SuppressWarnings("unchecked")
-    Coder<T> actualCoder = (Coder<T>) factory.create(Collections.<Coder<?>>emptyList());
-    assertEquals(expectedCoder, actualCoder);
-  }
-}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderProvidersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderProvidersTest.java
index 44be56daaff0..2aa8351c30fe 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderProvidersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderProvidersTest.java
@@ -17,54 +17,78 @@
  */
 package org.apache.beam.sdk.coders;
 
-import static org.hamcrest.Matchers.instanceOf;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
 
-import java.util.Map;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.junit.Rule;
+import org.apache.beam.sdk.values.TypeDescriptors;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /**
- * Tests for {@link CoderFactories}.
+ * Tests for {@link CoderProviders}.
  */
 @RunWith(JUnit4.class)
 public class CoderProvidersTest {
+  @Test
+  public void testCoderProvidersFromStaticMethodsForParameterlessTypes() throws Exception {
+    CoderProvider factory = CoderProviders.fromStaticMethods(String.class, StringUtf8Coder.class);
+    assertEquals(StringUtf8Coder.of(),
+        factory.coderFor(TypeDescriptors.strings(), Collections.<Coder<?>>emptyList()));
 
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
+    factory = CoderProviders.fromStaticMethods(Double.class, DoubleCoder.class);
+    assertEquals(DoubleCoder.of(),
+        factory.coderFor(TypeDescriptors.doubles(), Collections.<Coder<?>>emptyList()));
 
-  @Test
-  public void testAvroThenSerializableStringMap() throws Exception {
-    CoderProvider provider = CoderProviders.firstOf(AvroCoder.PROVIDER, SerializableCoder.PROVIDER);
-    Coder<Map<String, String>> coder =
-        provider.getCoder(new TypeDescriptor<Map<String, String>>(){});
-    assertThat(coder, instanceOf(AvroCoder.class));
+    factory = CoderProviders.fromStaticMethods(byte[].class, ByteArrayCoder.class);
+    assertEquals(ByteArrayCoder.of(),
+        factory.coderFor(TypeDescriptor.of(byte[].class), Collections.<Coder<?>>emptyList()));
   }
 
+  /**
+   * Checks that {#link CoderProviders.fromStaticMethods} successfully
+   * builds a working {@link CoderProvider} from {@link KvCoder KvCoder.class}.
+   */
   @Test
-  public void testThrowingThenSerializable() throws Exception {
-    CoderProvider provider =
-        CoderProviders.firstOf(new ThrowingCoderProvider(), SerializableCoder.PROVIDER);
-    Coder<Integer> coder = provider.getCoder(new TypeDescriptor<Integer>(){});
-    assertThat(coder, instanceOf(SerializableCoder.class));
+  public void testKvCoderProvider() throws Exception {
+    TypeDescriptor<KV<Double, Double>> type =
+        TypeDescriptors.kvs(TypeDescriptors.doubles(), TypeDescriptors.doubles());
+    CoderProvider kvCoderProvider = CoderProviders.fromStaticMethods(KV.class, KvCoder.class);
+    assertEquals(
+        KvCoder.of(DoubleCoder.of(), DoubleCoder.of()),
+        kvCoderProvider.coderFor(type, Arrays.asList(DoubleCoder.of(), DoubleCoder.of())));
   }
 
+  /**
+   * Checks that {#link CoderProviders.fromStaticMethods} successfully
+   * builds a working {@link CoderProvider} from {@link ListCoder ListCoder.class}.
+   */
   @Test
-  public void testNullThrows() throws Exception {
-    CoderProvider provider = CoderProviders.firstOf(new ThrowingCoderProvider());
-    thrown.expect(CannotProvideCoderException.class);
-    thrown.expectMessage("ThrowingCoderProvider");
-    provider.getCoder(new TypeDescriptor<Integer>(){});
+  public void testListCoderProvider() throws Exception {
+    TypeDescriptor<List<Double>> type = TypeDescriptors.lists(TypeDescriptors.doubles());
+    CoderProvider listCoderProvider = CoderProviders.fromStaticMethods(List.class, ListCoder.class);
+
+    assertEquals(
+        ListCoder.of(DoubleCoder.of()),
+        listCoderProvider.coderFor(type, Arrays.asList(DoubleCoder.of())));
   }
 
-  private static class ThrowingCoderProvider implements CoderProvider {
-    @Override
-    public <T> Coder<T> getCoder(TypeDescriptor<T> type) throws CannotProvideCoderException {
-      throw new CannotProvideCoderException("ThrowingCoderProvider cannot ever provide a Coder");
-    }
+  /**
+   * Checks that {#link CoderProviders.fromStaticMethods} successfully
+   * builds a working {@link CoderProvider} from {@link IterableCoder IterableCoder.class}.
+   */
+  @Test
+  public void testIterableCoderProvider() throws Exception {
+    TypeDescriptor<Iterable<Double>> type = TypeDescriptors.iterables(TypeDescriptors.doubles());
+    CoderProvider iterableCoderProvider =
+        CoderProviders.fromStaticMethods(Iterable.class, IterableCoder.class);
+
+    assertEquals(
+        IterableCoder.of(DoubleCoder.of()),
+        iterableCoderProvider.coderFor(type, Arrays.asList(DoubleCoder.of())));
   }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
index 5107355e0ea8..9568324ad3b4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
@@ -23,13 +23,11 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.lang.reflect.Type;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -73,29 +71,11 @@ private static class SerializableClass implements Serializable {
 
   private static class NotSerializableClass { }
 
-  @Test
-  public void testSerializableFallbackCoderProvider() throws Exception {
-    CoderRegistry registry = CoderRegistry.createDefault();
-    registry.setFallbackCoderProvider(SerializableCoder.PROVIDER);
-    Coder<?> serializableCoder = registry.getDefaultCoder(SerializableClass.class);
-
-    assertEquals(serializableCoder, SerializableCoder.of(SerializableClass.class));
-  }
-
-  @Test
-  public void testAvroFallbackCoderProvider() throws Exception {
-    CoderRegistry registry = CoderRegistry.createDefault();
-    registry.setFallbackCoderProvider(AvroCoder.PROVIDER);
-    Coder<?> avroCoder = registry.getDefaultCoder(NotSerializableClass.class);
-
-    assertEquals(avroCoder, AvroCoder.of(NotSerializableClass.class));
-  }
-
   @Test
   public void testRegisterInstantiatedCoder() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
-    registry.registerCoder(MyValue.class, MyValueCoder.of());
-    assertEquals(registry.getDefaultCoder(MyValue.class), MyValueCoder.of());
+    registry.registerCoderForClass(MyValue.class, MyValueCoder.of());
+    assertEquals(registry.getCoder(MyValue.class), MyValueCoder.of());
   }
 
   @SuppressWarnings("rawtypes") // this class exists to fail a test because of its rawtypes
@@ -120,18 +100,10 @@ public List<Coder<?>> getCoderArguments() {
     public void verifyDeterministic() throws NonDeterministicException {}
   }
 
-  @Test
-  public void testRegisterInstantiatedCoderInvalidRawtype() throws Exception {
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("may not be used with unspecialized generic classes");
-    CoderRegistry registry = CoderRegistry.createDefault();
-    registry.registerCoder(List.class, new MyListCoder());
-  }
-
   @Test
   public void testSimpleDefaultCoder() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
-    assertEquals(StringUtf8Coder.of(), registry.getDefaultCoder(String.class));
+    assertEquals(StringUtf8Coder.of(), registry.getCoder(String.class));
   }
 
   @Test
@@ -139,11 +111,9 @@ public void testSimpleUnknownDefaultCoder() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
     thrown.expect(CannotProvideCoderException.class);
     thrown.expectMessage(allOf(
-        containsString(UnknownType.class.getCanonicalName()),
-        containsString("No CoderFactory has been registered"),
-        containsString("does not have a @DefaultCoder annotation"),
-        containsString("does not implement Serializable")));
-    registry.getDefaultCoder(UnknownType.class);
+        containsString(UnknownType.class.getName()),
+        containsString("Unable to provide a Coder for")));
+    registry.getCoder(UnknownType.class);
   }
 
   @Test
@@ -151,14 +121,15 @@ public void testParameterizedDefaultListCoder() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
     TypeDescriptor<List<Integer>> listToken = new TypeDescriptor<List<Integer>>() {};
     assertEquals(ListCoder.of(VarIntCoder.of()),
-                 registry.getDefaultCoder(listToken));
+                 registry.getCoder(listToken));
 
-    registry.registerCoder(MyValue.class, MyValueCoder.class);
+    registry.registerCoderProvider(
+        CoderProviders.fromStaticMethods(MyValue.class, MyValueCoder.class));
     TypeDescriptor<KV<String, List<MyValue>>> kvToken =
         new TypeDescriptor<KV<String, List<MyValue>>>() {};
     assertEquals(KvCoder.of(StringUtf8Coder.of(),
                             ListCoder.of(MyValueCoder.of())),
-                 registry.getDefaultCoder(kvToken));
+                 registry.getCoder(kvToken));
 
   }
 
@@ -167,7 +138,7 @@ public void testParameterizedDefaultMapCoder() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
     TypeDescriptor<Map<Integer, String>> mapToken = new TypeDescriptor<Map<Integer, String>>() {};
     assertEquals(MapCoder.of(VarIntCoder.of(), StringUtf8Coder.of()),
-                 registry.getDefaultCoder(mapToken));
+                 registry.getCoder(mapToken));
   }
 
   @Test
@@ -177,21 +148,21 @@ public void testParameterizedDefaultNestedMapCoder() throws Exception {
         new TypeDescriptor<Map<Integer, Map<String, Double>>>() {};
     assertEquals(
         MapCoder.of(VarIntCoder.of(), MapCoder.of(StringUtf8Coder.of(), DoubleCoder.of())),
-        registry.getDefaultCoder(mapToken));
+        registry.getCoder(mapToken));
   }
 
   @Test
   public void testParameterizedDefaultSetCoder() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
     TypeDescriptor<Set<Integer>> setToken = new TypeDescriptor<Set<Integer>>() {};
-    assertEquals(SetCoder.of(VarIntCoder.of()), registry.getDefaultCoder(setToken));
+    assertEquals(SetCoder.of(VarIntCoder.of()), registry.getCoder(setToken));
   }
 
   @Test
   public void testParameterizedDefaultNestedSetCoder() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
     TypeDescriptor<Set<Set<Integer>>> setToken = new TypeDescriptor<Set<Set<Integer>>>() {};
-    assertEquals(SetCoder.of(SetCoder.of(VarIntCoder.of())), registry.getDefaultCoder(setToken));
+    assertEquals(SetCoder.of(SetCoder.of(VarIntCoder.of())), registry.getCoder(setToken));
   }
 
   @Test
@@ -201,11 +172,11 @@ public void testParameterizedDefaultCoderUnknown() throws Exception {
 
     thrown.expect(CannotProvideCoderException.class);
     thrown.expectMessage(String.format(
-        "Cannot provide coder for parameterized type %s: Unable to provide a default Coder for %s",
+        "Cannot provide coder for parameterized type %s: Unable to provide a Coder for %s",
         listUnknownToken,
-        UnknownType.class.getCanonicalName()));
+        UnknownType.class.getName()));
 
-    registry.getDefaultCoder(listUnknownToken);
+    registry.getCoder(listUnknownToken);
   }
 
   @Test
@@ -214,7 +185,7 @@ public void testTypeParameterInferenceForward() throws Exception {
     MyGenericClass<MyValue, List<MyValue>> instance =
         new MyGenericClass<MyValue, List<MyValue>>() {};
 
-    Coder<?> bazCoder = registry.getDefaultCoder(
+    Coder<?> bazCoder = registry.getCoder(
         instance.getClass(),
         MyGenericClass.class,
         Collections.<Type, Coder<?>>singletonMap(
@@ -230,7 +201,7 @@ public void testTypeParameterInferenceBackward() throws Exception {
     MyGenericClass<MyValue, List<MyValue>> instance =
         new MyGenericClass<MyValue, List<MyValue>>() {};
 
-    Coder<?> fooCoder = registry.getDefaultCoder(
+    Coder<?> fooCoder = registry.getCoder(
         instance.getClass(),
         MyGenericClass.class,
         Collections.<Type, Coder<?>>singletonMap(
@@ -241,49 +212,6 @@ public void testTypeParameterInferenceBackward() throws Exception {
     assertEquals(MyValueCoder.of(), fooCoder);
   }
 
-  @Test
-  public void testGetDefaultCoderFromIntegerValue() throws Exception {
-    CoderRegistry registry = CoderRegistry.createDefault();
-    Integer i = 13;
-    Coder<Integer> coder = registry.getDefaultCoder(i);
-    assertEquals(VarIntCoder.of(), coder);
-  }
-
-  @Test
-  public void testGetDefaultCoderFromNullValue() throws Exception {
-    CoderRegistry registry = CoderRegistry.createDefault();
-    assertEquals(VoidCoder.of(), registry.getDefaultCoder((Void) null));
-  }
-
-  @Test
-  public void testGetDefaultCoderFromKvValue() throws Exception {
-    CoderRegistry registry = CoderRegistry.createDefault();
-    KV<Integer, String> kv = KV.of(13, "hello");
-    Coder<KV<Integer, String>> coder = registry.getDefaultCoder(kv);
-    assertEquals(KvCoder.of(VarIntCoder.of(), StringUtf8Coder.of()),
-        coder);
-  }
-
-  @Test
-  public void testGetDefaultCoderFromKvNullValue() throws Exception {
-    CoderRegistry registry = CoderRegistry.createDefault();
-    KV<Void, Void> kv = KV.of((Void) null, (Void) null);
-    assertEquals(KvCoder.of(VoidCoder.of(), VoidCoder.of()),
-        registry.getDefaultCoder(kv));
-  }
-
-  @Test
-  public void testGetDefaultCoderFromNestedKvValue() throws Exception {
-    CoderRegistry registry = CoderRegistry.createDefault();
-    KV<Integer, KV<Long, KV<String, String>>> kv = KV.of(13, KV.of(17L, KV.of("hello", "goodbye")));
-    Coder<KV<Integer, KV<Long, KV<String, String>>>> coder = registry.getDefaultCoder(kv);
-    assertEquals(
-        KvCoder.of(VarIntCoder.of(),
-            KvCoder.of(VarLongCoder.of(),
-                KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))),
-        coder);
-  }
-
   @Test
   public void testTypeCompatibility() throws Exception {
     CoderRegistry.verifyCompatible(BigEndianIntegerCoder.of(), Integer.class);
@@ -335,7 +263,7 @@ public void testComponentIncompatibility() throws Exception {
   public void testDefaultCoderAnnotationGenericRawtype() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
     assertEquals(
-        registry.getDefaultCoder(MySerializableGeneric.class),
+        registry.getCoder(MySerializableGeneric.class),
         SerializableCoder.of(MySerializableGeneric.class));
   }
 
@@ -343,7 +271,7 @@ public void testDefaultCoderAnnotationGenericRawtype() throws Exception {
   public void testDefaultCoderAnnotationGeneric() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
     assertEquals(
-        registry.getDefaultCoder(new TypeDescriptor<MySerializableGeneric<String>>() {}),
+        registry.getCoder(new TypeDescriptor<MySerializableGeneric<String>>() {}),
         SerializableCoder.of(MySerializableGeneric.class));
   }
 
@@ -371,11 +299,8 @@ public void testTypeVariableErrorMessage() throws Exception {
     CoderRegistry registry = CoderRegistry.createDefault();
 
     thrown.expect(CannotProvideCoderException.class);
-    thrown.expectMessage(allOf(
-        containsString("No CoderFactory has been registered"),
-        containsString("does not have a @DefaultCoder annotation"),
-        containsString("does not implement Serializable")));
-    registry.getDefaultCoder(TypeDescriptor.of(
+    thrown.expectMessage("Unable to provide a Coder");
+    registry.getCoder(TypeDescriptor.of(
         TestGenericClass.class.getTypeParameters()[0]));
   }
 
@@ -388,8 +313,7 @@ public void testSerializableTypeVariableDefaultCoder() throws Exception {
 
     TypeDescriptor type = TypeDescriptor.of(
         TestSerializableGenericClass.class.getTypeParameters()[0]);
-    assertEquals(registry.getDefaultCoder(type),
-        SerializableCoder.of(type));
+    assertEquals(SerializableCoder.of(type), registry.getCoder(type));
   }
 
   private static class TestSerializableGenericClass<TestGenericT extends Serializable> {}
@@ -450,12 +374,6 @@ public static MyValueCoder of() {
       return INSTANCE;
     }
 
-    @SuppressWarnings("unused")
-    public static List<Object> getInstanceComponents(
-        @SuppressWarnings("unused") MyValue exampleValue) {
-      return Arrays.asList();
-    }
-
     @Override
     public void encode(MyValue value, OutputStream outStream, Context context)
         throws CoderException, IOException {
@@ -498,7 +416,12 @@ public TypeDescriptor<MyValue> getEncodedTypeDescriptor() {
     }
   }
 
-  private static class UnknownType { }
+  /**
+   * This type is incompatible with all known coder providers such as Serializable,
+   * {@code @DefaultCoder} which allows testing scenarios where coder lookup fails.
+   */
+  private static class UnknownType {
+  }
 
   @DefaultCoder(SerializableCoder.class)
   private static class MySerializableGeneric<T extends Serializable> implements Serializable {
@@ -506,20 +429,34 @@ private static class MySerializableGeneric<T extends Serializable> implements Se
     private T foo;
   }
 
+  /**
+   * This type is incompatible with all known coder providers such as Serializable,
+   * {@code @DefaultCoder} which allows testing the automatic registration mechanism.
+   */
+  private static class AutoRegistrationClass {
+  }
+
+  private static class AutoRegistrationClassCoder extends CustomCoder<AutoRegistrationClass> {
+    private static final AutoRegistrationClassCoder INSTANCE = new AutoRegistrationClassCoder();
+  }
+
   @Test
-  public void testAutomaticRegistrationOfCoders() throws Exception {
-    assertEquals(CoderRegistry.createDefault().getDefaultCoder(MyValue.class), MyValueCoder.of());
+  public void testAutomaticRegistrationOfCoderProviders() throws Exception {
+    assertEquals(AutoRegistrationClassCoder.INSTANCE,
+        CoderRegistry.createDefault().getCoder(AutoRegistrationClass.class));
   }
 
   /**
-   * A {@link CoderRegistrar} to demonstrate default {@link Coder} registration.
+   * A {@link CoderProviderRegistrar} to demonstrate default {@link Coder} registration.
    */
-  @AutoService(CoderRegistrar.class)
-  public static class RegisteredTestCoderRegistrar implements CoderRegistrar {
+  @AutoService(CoderProviderRegistrar.class)
+  public static class RegisteredTestCoderProviderRegistrar implements CoderProviderRegistrar {
     @Override
-    public Map<Class<?>, CoderFactory> getCoderFactoriesToUseForClasses() {
-      return ImmutableMap.<Class<?>, CoderFactory>of(
-          MyValue.class, CoderFactories.forCoder(MyValueCoder.of()));
+    public List<CoderProvider> getCoderProviders() {
+      return ImmutableList.of(
+          CoderProviders.forCoder(
+              TypeDescriptor.of(AutoRegistrationClass.class),
+              AutoRegistrationClassCoder.INSTANCE));
     }
   }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java
index d335b18cc584..aa8d94c2c1af 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java
@@ -18,13 +18,15 @@
 package org.apache.beam.sdk.coders;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertThat;
 
 import java.io.Serializable;
+import java.util.Collections;
 import java.util.List;
+import org.apache.beam.sdk.coders.DefaultCoder.DefaultCoderProviderRegistrar.DefaultCoderProvider;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.hamcrest.Matchers;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -32,15 +34,12 @@
 import org.junit.runners.JUnit4;
 
 /**
- * Tests of Coder defaults.
+ * Tests for {@link DefaultCoder}.
  */
 @RunWith(JUnit4.class)
 public class DefaultCoderTest {
 
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  public CoderRegistry registry = CoderRegistry.createDefault();
+  @Rule public ExpectedException thrown = ExpectedException.none();
 
   @DefaultCoder(AvroCoder.class)
   private static class AvroRecord {
@@ -75,6 +74,17 @@ public static <T extends Serializable> SerializableCoder<T> of(TypeDescriptor<T>
     protected CustomSerializableCoder() {
       super(CustomRecord.class, TypeDescriptor.of(CustomRecord.class));
     }
+
+    @SuppressWarnings("unused")
+    public static CoderProvider getCoderProvider() {
+      return new CoderProvider() {
+        @Override
+        public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+            List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+          return CustomSerializableCoder.of((TypeDescriptor) typeDescriptor);
+        }
+      };
+    }
   }
 
   private static class OldCustomSerializableCoder extends SerializableCoder<OldCustomRecord> {
@@ -89,33 +99,52 @@ public static <T extends Serializable> SerializableCoder<T> of(Class<T> recordTy
     protected OldCustomSerializableCoder() {
       super(OldCustomRecord.class, TypeDescriptor.of(OldCustomRecord.class));
     }
+
+    @SuppressWarnings("unused")
+    public static CoderProvider getCoderProvider() {
+      return new CoderProvider() {
+        @Override
+        public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+            List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+          return OldCustomSerializableCoder.of((Class) typeDescriptor.getRawType());
+        }
+      };
+    }
   }
 
   @Test
-  public void testDefaultCoderClasses() throws Exception {
-    assertThat(registry.getDefaultCoder(AvroRecord.class), instanceOf(AvroCoder.class));
-    assertThat(registry.getDefaultCoder(SerializableBase.class),
-        instanceOf(SerializableCoder.class));
-    assertThat(registry.getDefaultCoder(SerializableRecord.class),
+  public void testCodersWithoutComponents() throws Exception {
+    CoderRegistry registry = CoderRegistry.createDefault();
+    registry.registerCoderProvider(
+        new DefaultCoderProvider());
+    assertThat(registry.getCoder(AvroRecord.class),
+        instanceOf(AvroCoder.class));
+    assertThat(registry.getCoder(SerializableRecord.class),
         instanceOf(SerializableCoder.class));
-    assertThat(registry.getDefaultCoder(CustomRecord.class),
+    assertThat(registry.getCoder(CustomRecord.class),
         instanceOf(CustomSerializableCoder.class));
-    assertThat(registry.getDefaultCoder(OldCustomRecord.class),
+    assertThat(registry.getCoder(OldCustomRecord.class),
         instanceOf(OldCustomSerializableCoder.class));
   }
 
   @Test
   public void testDefaultCoderInCollection() throws Exception {
-    assertThat(registry.getDefaultCoder(new TypeDescriptor<List<AvroRecord>>(){}),
-        instanceOf(ListCoder.class));
-    assertThat(registry.getDefaultCoder(new TypeDescriptor<List<SerializableRecord>>(){}),
-        equalTo((Coder<List<SerializableRecord>>)
+    CoderRegistry registry = CoderRegistry.createDefault();
+    registry.registerCoderProvider(
+        new DefaultCoderProvider());
+    Coder<List<AvroRecord>> avroRecordCoder =
+        registry.getCoder(new TypeDescriptor<List<AvroRecord>>(){});
+    assertThat(avroRecordCoder, instanceOf(ListCoder.class));
+    assertThat(((ListCoder) avroRecordCoder).getElemCoder(), instanceOf(AvroCoder.class));
+    assertThat(registry.getCoder(new TypeDescriptor<List<SerializableRecord>>(){}),
+        Matchers.<Coder<List<SerializableRecord>>>equalTo(
             ListCoder.of(SerializableCoder.of(SerializableRecord.class))));
   }
 
   @Test
   public void testUnknown() throws Exception {
     thrown.expect(CannotProvideCoderException.class);
-    registry.getDefaultCoder(Unknown.class);
+    new DefaultCoderProvider().coderFor(
+        TypeDescriptor.of(Unknown.class), Collections.<Coder<?>>emptyList());
   }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/IterableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/IterableCoderTest.java
index 1e561358ede8..1141fb539d66 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/IterableCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/IterableCoderTest.java
@@ -18,8 +18,6 @@
 package org.apache.beam.sdk.coders;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 
 import java.util.Arrays;
@@ -61,21 +59,6 @@ public void testDecodeEncodeContentsInSameOrder() throws Exception {
     }
   }
 
-  @Test
-  public void testGetInstanceComponentsNonempty() {
-    Iterable<Integer> iterable = Arrays.asList(2, 58, 99, 5);
-    List<Object> components = IterableCoder.getInstanceComponents(iterable);
-    assertEquals(1, components.size());
-    assertEquals(2, components.get(0));
-  }
-
-  @Test
-  public void testGetInstanceComponentsEmpty() {
-    Iterable<Integer> iterable = Arrays.asList();
-    List<Object> components = IterableCoder.getInstanceComponents(iterable);
-    assertNull(components);
-  }
-
   @Test
   public void testCoderSerializable() throws Exception {
     CoderProperties.coderSerializable(TEST_CODER);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/ListCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/ListCoderTest.java
index 35239d68dd81..bcc44003a118 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/ListCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/ListCoderTest.java
@@ -18,8 +18,6 @@
 package org.apache.beam.sdk.coders;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 
 import java.util.Arrays;
@@ -61,21 +59,6 @@ public void testDecodeEncodeContentsInSameOrder() throws Exception {
     }
   }
 
-  @Test
-  public void testGetInstanceComponentsNonempty() throws Exception {
-    List<Integer> list = Arrays.asList(21, 5, 3, 5);
-    List<Object> components = ListCoder.getInstanceComponents(list);
-    assertEquals(1, components.size());
-    assertEquals(21, components.get(0));
-  }
-
-  @Test
-  public void testGetInstanceComponentsEmpty() throws Exception {
-    List<Integer> list = Arrays.asList();
-    List<Object> components = ListCoder.getInstanceComponents(list);
-    assertNull(components);
-  }
-
   @Test
   public void testEmptyList() throws Exception {
     List<Integer> list = Collections.emptyList();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java
index a52e6cb9a323..19c895e11891 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java
@@ -18,14 +18,11 @@
 package org.apache.beam.sdk.coders;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 
 import com.google.common.collect.ImmutableMap;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -64,23 +61,6 @@ public void testCoderIsSerializableWithWellKnownCoderType() throws Exception {
         MapCoder.of(GlobalWindow.Coder.INSTANCE, GlobalWindow.Coder.INSTANCE));
   }
 
-  @Test
-  public void testGetInstanceComponentsNonempty() {
-    Map<Integer, String> map = new HashMap<>();
-    map.put(17, "foozle");
-    List<Object> components = MapCoder.getInstanceComponents(map);
-    assertEquals(2, components.size());
-    assertEquals(17, components.get(0));
-    assertEquals("foozle", components.get(1));
-  }
-
-  @Test
-  public void testGetInstanceComponentsEmpty() {
-    Map<Integer, String> map = new HashMap<>();
-    List<Object> components = MapCoder.getInstanceComponents(map);
-    assertNull(components);
-  }
-
   /**
    * Generated data to check that the wire format has not changed. To regenerate, see
    * {@link org.apache.beam.sdk.coders.PrintBase64Encodings}.
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
index ef6df345e3fd..d97eea6b9ed4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.coders;
 
+import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
@@ -230,4 +231,14 @@ public void testEncodedTypeDescriptor() throws Exception {
         SerializableCoder.of(MyRecord.class).getEncodedTypeDescriptor(),
         Matchers.equalTo(TypeDescriptor.of(MyRecord.class)));
   }
+
+  private static class AutoRegistration implements Serializable {
+    private static final long serialVersionUID = 42L;
+  }
+
+  @Test
+  public void testSerializableCoderProviderIsRegistered() throws Exception {
+    assertThat(CoderRegistry.createDefault().getCoder(AutoRegistration.class),
+        instanceOf(SerializableCoder.class));
+  }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
index 62edac9b55d9..8a4d56365410 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
@@ -104,7 +104,7 @@ public void testDuplicatedTagsWithContext() {
   @Test
   @Category(ValidatesRunner.class)
   public void testComposedCombine() {
-    p.getCoderRegistry().registerCoder(UserString.class, UserStringCoder.of());
+    p.getCoderRegistry().registerCoderForClass(UserString.class, UserStringCoder.of());
 
     PCollection<KV<String, KV<Integer, UserString>>> perKeyInput = p.apply(
         Create.timestamped(
@@ -156,7 +156,7 @@ public void testComposedCombine() {
   @Test
   @Category(ValidatesRunner.class)
   public void testComposedCombineWithContext() {
-    p.getCoderRegistry().registerCoder(UserString.class, UserStringCoder.of());
+    p.getCoderRegistry().registerCoderForClass(UserString.class, UserStringCoder.of());
 
     PCollectionView<String> view = p
         .apply(Create.of("I"))
@@ -218,8 +218,10 @@ public void testComposedCombineWithContext() {
   @Test
   @Category(ValidatesRunner.class)
   public void testComposedCombineNullValues() {
-    p.getCoderRegistry().registerCoder(UserString.class, NullableCoder.of(UserStringCoder.of()));
-    p.getCoderRegistry().registerCoder(String.class, NullableCoder.of(StringUtf8Coder.of()));
+    p.getCoderRegistry().registerCoderForClass(
+        UserString.class, NullableCoder.of(UserStringCoder.of()));
+    p.getCoderRegistry().registerCoderForClass(
+        String.class, NullableCoder.of(StringUtf8Coder.of()));
 
     PCollection<KV<String, KV<Integer, UserString>>> perKeyInput = p.apply(
         Create.timestamped(
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
index 76f61b32dc67..a4588128604e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
@@ -321,7 +321,7 @@ public void testCreateTimestampedDefaultOutputCoderUsingCoder() throws Exception
   @Test
   public void testCreateTimestampedDefaultOutputCoderUsingTypeDescriptor() throws Exception {
     Coder<Record> coder = new RecordCoder();
-    p.getCoderRegistry().registerCoder(Record.class, coder);
+    p.getCoderRegistry().registerCoderForClass(Record.class, coder);
     PBegin pBegin = PBegin.in(p);
     Create.TimestampedValues<Record> values =
         Create.timestamped(
@@ -364,7 +364,7 @@ public void testCreateGetName() {
   @Test
   public void testCreateDefaultOutputCoderUsingInference() throws Exception {
     Coder<Record> coder = new RecordCoder();
-    p.getCoderRegistry().registerCoder(Record.class, coder);
+    p.getCoderRegistry().registerCoderForClass(Record.class, coder);
     PBegin pBegin = PBegin.in(p);
     Create.Values<Record> values = Create.of(new Record(), new Record(), new Record());
     Coder<Record> defaultCoder = values.getDefaultOutputCoder(pBegin);
@@ -384,7 +384,7 @@ public void testCreateDefaultOutputCoderUsingCoder() throws Exception {
   @Test
   public void testCreateDefaultOutputCoderUsingTypeDescriptor() throws Exception {
     Coder<Record> coder = new RecordCoder();
-    p.getCoderRegistry().registerCoder(Record.class, coder);
+    p.getCoderRegistry().registerCoderForClass(Record.class, coder);
     PBegin pBegin = PBegin.in(p);
     Create.Values<Record> values =
         Create.of(new Record(), new Record2()).withType(new TypeDescriptor<Record>() {});
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java
index b24071ec72cf..11f284f35538 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java
@@ -94,8 +94,8 @@ public Set<String> apply(String input) {
 
     assertThat(output.getTypeDescriptor(),
         equalTo((TypeDescriptor<String>) new TypeDescriptor<String>() {}));
-    assertThat(pipeline.getCoderRegistry().getDefaultCoder(output.getTypeDescriptor()),
-        equalTo(pipeline.getCoderRegistry().getDefaultCoder(new TypeDescriptor<String>() {})));
+    assertThat(pipeline.getCoderRegistry().getCoder(output.getTypeDescriptor()),
+        equalTo(pipeline.getCoderRegistry().getCoder(new TypeDescriptor<String>() {})));
 
     // Make sure the pipeline runs
     pipeline.run();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
index 6982e01116bc..aba33ebe8b5a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
@@ -41,6 +41,7 @@
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderProviders;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.MapCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -392,7 +393,8 @@ public void testGroupByKeyWithBadEqualsHashCode() throws Exception {
     final int numValues = 10;
     final int numKeys = 5;
 
-    p.getCoderRegistry().registerCoder(BadEqualityKey.class, DeterministicKeyCoder.class);
+    p.getCoderRegistry().registerCoderProvider(
+        CoderProviders.fromStaticMethods(BadEqualityKey.class, DeterministicKeyCoder.class));
 
     // construct input data
     List<KV<BadEqualityKey, Long>> input = new ArrayList<>();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
index 7bf94a058356..241b60e35632 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
@@ -187,8 +187,8 @@ public String apply(String input) {
         }));
     assertThat(output.getTypeDescriptor(),
         equalTo((TypeDescriptor<String>) new TypeDescriptor<String>() {}));
-    assertThat(pipeline.getCoderRegistry().getDefaultCoder(output.getTypeDescriptor()),
-        equalTo(pipeline.getCoderRegistry().getDefaultCoder(new TypeDescriptor<String>() {})));
+    assertThat(pipeline.getCoderRegistry().getCoder(output.getTypeDescriptor()),
+        equalTo(pipeline.getCoderRegistry().getCoder(new TypeDescriptor<String>() {})));
 
     // Make sure the pipeline runs too
     pipeline.run();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 073957f30001..cbbe7f13963c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -985,11 +985,6 @@ public static TestDummyCoder of() {
       return INSTANCE;
     }
 
-    @SuppressWarnings("unused") // used to create a CoderFactory
-    public static List<Object> getInstanceComponents(TestDummy exampleValue) {
-      return Collections.emptyList();
-    }
-
     @Override
     public void encode(TestDummy value, OutputStream outStream, Context context)
         throws CoderException, IOException {
@@ -1657,7 +1652,7 @@ public void processElement(
   public void testValueStateCoderInference() {
     final String stateId = "foo";
     MyIntegerCoder myIntegerCoder = MyIntegerCoder.of();
-    pipeline.getCoderRegistry().registerCoder(MyInteger.class, myIntegerCoder);
+    pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
 
     DoFn<KV<String, Integer>, MyInteger> fn =
         new DoFn<KV<String, Integer>, MyInteger>() {
@@ -1750,7 +1745,7 @@ public void processElement(
   public void testCoderInferenceOfList() {
     final String stateId = "foo";
     MyIntegerCoder myIntegerCoder = MyIntegerCoder.of();
-    pipeline.getCoderRegistry().registerCoder(MyInteger.class, myIntegerCoder);
+    pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
 
     DoFn<KV<String, Integer>, List<MyInteger>> fn =
         new DoFn<KV<String, Integer>, List<MyInteger>>() {
@@ -1966,7 +1961,7 @@ public void processElement(
   public void testBagStateCoderInference() {
     final String stateId = "foo";
     Coder<MyInteger> myIntegerCoder = MyIntegerCoder.of();
-    pipeline.getCoderRegistry().registerCoder(MyInteger.class, myIntegerCoder);
+    pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
 
     DoFn<KV<String, Integer>, List<MyInteger>> fn =
         new DoFn<KV<String, Integer>, List<MyInteger>>() {
@@ -2085,7 +2080,7 @@ public void testSetStateCoderInference() {
     final String stateId = "foo";
     final String countStateId = "count";
     Coder<MyInteger> myIntegerCoder = MyIntegerCoder.of();
-    pipeline.getCoderRegistry().registerCoder(MyInteger.class, myIntegerCoder);
+    pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
 
     DoFn<KV<String, Integer>, Set<MyInteger>> fn =
         new DoFn<KV<String, Integer>, Set<MyInteger>>() {
@@ -2217,7 +2212,7 @@ public void testMapStateCoderInference() {
     final String stateId = "foo";
     final String countStateId = "count";
     Coder<MyInteger> myIntegerCoder = MyIntegerCoder.of();
-    pipeline.getCoderRegistry().registerCoder(MyInteger.class, myIntegerCoder);
+    pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
 
     DoFn<KV<String, KV<String, Integer>>, KV<String, MyInteger>> fn =
         new DoFn<KV<String, KV<String, Integer>>, KV<String, MyInteger>>() {
@@ -2345,7 +2340,7 @@ public void processElement(
   @Test
   @Category({ValidatesRunner.class, UsesStatefulParDo.class})
   public void testCombiningStateCoderInference() {
-    pipeline.getCoderRegistry().registerCoder(MyInteger.class, MyIntegerCoder.of());
+    pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, MyIntegerCoder.of());
 
     final String stateId = "foo";
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index f26dd59ba9cc..489493ab22a8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -36,6 +36,7 @@
 import java.util.Arrays;
 import java.util.List;
 import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.CoderProviders;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.state.StateSpec;
@@ -422,7 +423,8 @@ public RestrictionWithDefaultTracker getInitialRestriction(String element) {
     DoFnInvoker<String, String> invoker = DoFnInvokers.invokerFor(fn);
 
     CoderRegistry coderRegistry = CoderRegistry.createDefault();
-    coderRegistry.registerCoder(RestrictionWithDefaultTracker.class, CoderForDefaultTracker.class);
+    coderRegistry.registerCoderProvider(CoderProviders.fromStaticMethods(
+        RestrictionWithDefaultTracker.class, CoderForDefaultTracker.class));
     assertThat(
         invoker.<RestrictionWithDefaultTracker>invokeGetRestrictionCoder(coderRegistry),
         instanceOf(CoderForDefaultTracker.class));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypedPValueTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypedPValueTest.java
index f752b1cd80b0..e1eb2fbada90 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypedPValueTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypedPValueTest.java
@@ -22,6 +22,7 @@
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertThat;
 
+import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
@@ -77,14 +78,10 @@ public void testUntypedOutputTupleTagGivesActionableMessage() {
 
     thrown.expect(IllegalStateException.class);
     thrown.expectMessage("No Coder has been manually specified");
-    thrown.expectMessage(
-        containsString("Building a Coder using a registered CoderFactory failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the @DefaultCoder annotation failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the fallback CoderProvider failed"));
+    thrown.expectMessage("Building a Coder using a registered CoderProvider failed");
 
-    tuple.get(untypedOutputTag).getCoder();
+    Coder<?> coder = tuple.get(untypedOutputTag).getCoder();
+    System.out.println(coder);
   }
 
   @Test
@@ -96,12 +93,7 @@ public void testStaticFactoryOutputTupleTagGivesActionableMessage() {
 
     thrown.expect(IllegalStateException.class);
     thrown.expectMessage("No Coder has been manually specified");
-    thrown.expectMessage(
-        containsString("Building a Coder using a registered CoderFactory failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the @DefaultCoder annotation failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the fallback CoderProvider failed"));
+    thrown.expectMessage("Building a Coder using a registered CoderProvider failed");
 
     tuple.get(untypedOutputTag).getCoder();
   }
@@ -126,7 +118,10 @@ public void testUntypedMainOutputTagTypedOutputTupleTag() {
     assertThat(tuple.get(typedOutputTag).getCoder(), instanceOf(VarIntCoder.class));
   }
 
-  // A simple class for which there should be no obvious Coder.
+  /**
+   * This type is incompatible with all known coder providers such as Serializable,
+   * {@code @DefaultCoder} which allows testing coder registry lookup failure cases.
+   */
   static class EmptyClass {
   }
 
@@ -149,10 +144,7 @@ public void testParDoWithNoOutputsErrorDoesNotMentionTupleTag() {
     thrown.expectMessage(not(containsString("erasure")));
     thrown.expectMessage(not(containsString("see TupleTag Javadoc")));
     // Instead, expect output suggesting other possible fixes.
-    thrown.expectMessage(containsString("Building a Coder using a registered CoderFactory failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the @DefaultCoder annotation failed"));
-    thrown.expectMessage(containsString("Building a Coder from the fallback CoderProvider failed"));
+    thrown.expectMessage("Building a Coder using a registered CoderProvider failed");
 
     input.getCoder();
   }
diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java
index 9577c6eedb3b..968a2fa06230 100644
--- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java
+++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java
@@ -31,6 +31,7 @@
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.SortedSet;
@@ -41,6 +42,7 @@
 import org.apache.beam.sdk.coders.CoderProvider;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.DefaultCoder;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
@@ -105,13 +107,6 @@
  */
 public class ProtoCoder<T extends Message> extends CustomCoder<T> {
 
-  /**
-   * A {@link CoderProvider} that returns a {@link ProtoCoder} with an empty
-   * {@link ExtensionRegistry}.
-   */
-  public static CoderProvider coderProvider() {
-    return PROVIDER;
-  }
 
   /**
    * Returns a {@link ProtoCoder} for the given Protocol Buffers {@link Message}.
@@ -291,36 +286,47 @@ private Parser<T> getParser() {
     return memoizedParser;
   }
 
-  static final TypeDescriptor<Message> CHECK = new TypeDescriptor<Message>() {};
+  /**
+   * Returns a {@link CoderProvider} which uses the {@link ProtoCoder} for
+   * {@link Message proto messages}.
+   *
+   * <p>This method is invoked reflectively from {@link DefaultCoder}.
+   */
+  public static CoderProvider getCoderProvider() {
+    return new ProtoCoderProvider();
+  }
+
+  static final TypeDescriptor<Message> MESSAGE_TYPE = new TypeDescriptor<Message>() {};
 
   /**
-   * The implementation of the {@link CoderProvider} for this {@link ProtoCoder} returned by
-   * {@link #coderProvider()}.
+   * A {@link CoderProvider} for {@link Message proto messages}.
    */
-  private static final CoderProvider PROVIDER =
-      new CoderProvider() {
-        @Override
-        public <T> Coder<T> getCoder(TypeDescriptor<T> type) throws CannotProvideCoderException {
-          if (!type.isSubtypeOf(CHECK)) {
-            throw new CannotProvideCoderException(
-                String.format(
-                    "Cannot provide %s because %s is not a subclass of %s",
-                    ProtoCoder.class.getSimpleName(),
-                    type,
-                    Message.class.getName()));
-          }
+  private static class ProtoCoderProvider extends CoderProvider {
 
-          @SuppressWarnings("unchecked")
-          TypeDescriptor<? extends Message> messageType = (TypeDescriptor<? extends Message>) type;
-          try {
-            @SuppressWarnings("unchecked")
-            Coder<T> coder = (Coder<T>) ProtoCoder.of(messageType);
-            return coder;
-          } catch (IllegalArgumentException e) {
-            throw new CannotProvideCoderException(e);
-          }
-        }
-      };
+    @Override
+    public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+        List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+      if (!typeDescriptor.isSubtypeOf(MESSAGE_TYPE)) {
+        throw new CannotProvideCoderException(
+            String.format(
+                "Cannot provide %s because %s is not a subclass of %s",
+                ProtoCoder.class.getSimpleName(),
+                typeDescriptor,
+                Message.class.getName()));
+      }
+
+      @SuppressWarnings("unchecked")
+      TypeDescriptor<? extends Message> messageType =
+          (TypeDescriptor<? extends Message>) typeDescriptor;
+      try {
+        @SuppressWarnings("unchecked")
+        Coder<T> coder = (Coder<T>) ProtoCoder.of(messageType);
+        return coder;
+      } catch (IllegalArgumentException e) {
+        throw new CannotProvideCoderException(e);
+      }
+    }
+  }
 
   private SortedSet<String> getSortedExtensionClasses() {
     SortedSet<String> ret = new TreeSet<>();
diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderRegistrar.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderProviderRegistrar.java
similarity index 58%
rename from sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderRegistrar.java
rename to sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderProviderRegistrar.java
index fb39a14dbe34..1cb79d733f83 100644
--- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderRegistrar.java
+++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderProviderRegistrar.java
@@ -18,22 +18,24 @@
 package org.apache.beam.sdk.extensions.protobuf;
 
 import com.google.auto.service.AutoService;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableList;
 import com.google.protobuf.ByteString;
-import java.util.Map;
-import org.apache.beam.sdk.coders.CoderFactories;
-import org.apache.beam.sdk.coders.CoderFactory;
-import org.apache.beam.sdk.coders.CoderRegistrar;
+import java.util.List;
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.coders.CoderProviderRegistrar;
+import org.apache.beam.sdk.coders.CoderProviders;
+import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
- * A {@link CoderRegistrar} for standard types used with Google Protobuf.
+ * A {@link CoderProviderRegistrar} for standard types used with Google Protobuf.
  */
-@AutoService(CoderRegistrar.class)
-public class ProtobufCoderRegistrar implements CoderRegistrar {
+@AutoService(CoderProviderRegistrar.class)
+public class ProtobufCoderProviderRegistrar implements CoderProviderRegistrar {
   @Override
-  public Map<Class<?>, CoderFactory> getCoderFactoriesToUseForClasses() {
-    return ImmutableMap.<Class<?>, CoderFactory>of(
-        ByteString.class, CoderFactories.forCoder(ByteStringCoder.of()));
+  public List<CoderProvider> getCoderProviders() {
+    return ImmutableList.of(
+        CoderProviders.forCoder(TypeDescriptor.of(ByteString.class), ByteStringCoder.of()),
+        ProtoCoder.getCoderProvider());
   }
 }
 
diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java
index 6d00b86633e0..d79bf4c70ad0 100644
--- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java
+++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java
@@ -21,6 +21,7 @@
 import static org.junit.Assert.assertNotEquals;
 
 import com.google.common.collect.ImmutableList;
+import java.util.Collections;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
@@ -53,7 +54,8 @@ public void testFactoryMethodAgreement() throws Exception {
 
     assertEquals(
         ProtoCoder.of(new TypeDescriptor<MessageA>() {}),
-        ProtoCoder.coderProvider().getCoder(new TypeDescriptor<MessageA>() {}));
+        ProtoCoder.getCoderProvider().coderFor(
+            new TypeDescriptor<MessageA>() {}, Collections.<Coder<?>>emptyList()));
   }
 
   @Test
@@ -61,7 +63,8 @@ public void testProviderCannotProvideCoder() throws Exception {
     thrown.expect(CannotProvideCoderException.class);
     thrown.expectMessage("java.lang.Integer is not a subclass of com.google.protobuf.Message");
 
-    ProtoCoder.coderProvider().getCoder(new TypeDescriptor<Integer>() {});
+    ProtoCoder.getCoderProvider().coderFor(
+        new TypeDescriptor<Integer>() {}, Collections.<Coder<?>>emptyList());
   }
 
   @Test
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderRegistrar.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrar.java
similarity index 56%
rename from sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderRegistrar.java
rename to sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrar.java
index 847c7b539756..c6fd849a2964 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderRegistrar.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrar.java
@@ -19,21 +19,22 @@
 
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.auto.service.AutoService;
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.apache.beam.sdk.coders.CoderFactories;
-import org.apache.beam.sdk.coders.CoderFactory;
-import org.apache.beam.sdk.coders.CoderRegistrar;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.coders.CoderProviderRegistrar;
+import org.apache.beam.sdk.coders.CoderProviders;
+import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
- * A {@link CoderRegistrar} for standard types used with {@link BigQueryIO}.
+ * A {@link CoderProviderRegistrar} for standard types used with {@link BigQueryIO}.
  */
-@AutoService(CoderRegistrar.class)
-public class BigQueryCoderRegistrar implements CoderRegistrar {
+@AutoService(CoderProviderRegistrar.class)
+public class BigQueryCoderProviderRegistrar implements CoderProviderRegistrar {
   @Override
-  public Map<Class<?>, CoderFactory> getCoderFactoriesToUseForClasses() {
-    return ImmutableMap.of(
-        TableRow.class, CoderFactories.forCoder(TableRowJsonCoder.of()),
-        TableRowInfo.class, CoderFactories.forCoder(TableRowInfoCoder.of()));
+  public List<CoderProvider> getCoderProviders() {
+    return ImmutableList.of(
+        CoderProviders.forCoder(TypeDescriptor.of(TableRow.class), TableRowJsonCoder.of()),
+        CoderProviders.forCoder(TypeDescriptor.of(TableRowInfo.class), TableRowInfoCoder.of()));
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java
index dc8bcff6978b..edb1e0d982e0 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java
@@ -168,7 +168,7 @@ Coder<DestinationT> getDestinationCoderWithDefault(CoderRegistry registry)
           Type parameter = parameterized.getActualTypeArguments()[1];
           @SuppressWarnings("unchecked")
           Class<DestinationT> parameterClass = (Class<DestinationT>) parameter;
-          return registry.getDefaultCoder(parameterClass);
+          return registry.getCoder(parameterClass);
         }
       }
     }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderRegistrar.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderProviderRegistrar.java
similarity index 56%
rename from sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderRegistrar.java
rename to sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderProviderRegistrar.java
index 062f3500e582..57bc903ea680 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderRegistrar.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderProviderRegistrar.java
@@ -18,18 +18,20 @@
 package org.apache.beam.sdk.io.gcp.pubsub;
 
 import com.google.auto.service.AutoService;
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.apache.beam.sdk.coders.CoderFactories;
-import org.apache.beam.sdk.coders.CoderFactory;
-import org.apache.beam.sdk.coders.CoderRegistrar;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.coders.CoderProviderRegistrar;
+import org.apache.beam.sdk.coders.CoderProviders;
+import org.apache.beam.sdk.values.TypeDescriptor;
 
-/** A {@link CoderRegistrar} for standard types used with {@link PubsubIO}. */
-@AutoService(CoderRegistrar.class)
-public class PubsubCoderRegistrar implements CoderRegistrar {
+/** A {@link CoderProviderRegistrar} for standard types used with {@link PubsubIO}. */
+@AutoService(CoderProviderRegistrar.class)
+public class PubsubCoderProviderRegistrar implements CoderProviderRegistrar {
   @Override
-  public Map<Class<?>, CoderFactory> getCoderFactoriesToUseForClasses() {
-    return ImmutableMap.<Class<?>, CoderFactory>of(
-        PubsubMessage.class, CoderFactories.forCoder(PubsubMessageWithAttributesCoder.of()));
+  public List<CoderProvider> getCoderProviders() {
+    return ImmutableList.of(
+        CoderProviders.forCoder(TypeDescriptor.of(PubsubMessage.class),
+            PubsubMessageWithAttributesCoder.of()));
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderRegistrarTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrarTest.java
similarity index 84%
rename from sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderRegistrarTest.java
rename to sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrarTest.java
index e7e9fe171bf2..bd5246a07f45 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderRegistrarTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrarTest.java
@@ -24,17 +24,17 @@
 import org.junit.runners.JUnit4;
 
 /**
- * Tests for {@link BigQueryCoderRegistrar}.
+ * Tests for {@link BigQueryCoderProviderRegistrar}.
  */
 @RunWith(JUnit4.class)
-public class BigQueryCoderRegistrarTest {
+public class BigQueryCoderProviderRegistrarTest {
   @Test
   public void testTableRowCoderIsRegistered() throws Exception {
-    CoderRegistry.createDefault().getDefaultCoder(TableRow.class);
+    CoderRegistry.createDefault().getCoder(TableRow.class);
   }
 
   @Test
   public void testTableRowInfoCoderIsRegistered() throws Exception {
-    CoderRegistry.createDefault().getDefaultCoder(TableRowInfo.class);
+    CoderRegistry.createDefault().getCoder(TableRowInfo.class);
   }
 }
diff --git a/sdks/java/io/hadoop-common/pom.xml b/sdks/java/io/hadoop-common/pom.xml
index ebb4b391ece6..9054f99a7bdf 100644
--- a/sdks/java/io/hadoop-common/pom.xml
+++ b/sdks/java/io/hadoop-common/pom.xml
@@ -77,5 +77,12 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <!-- build dependencies -->
+    <dependency>
+      <groupId>com.google.auto.service</groupId>
+      <artifactId>auto-service</artifactId>
+      <optional>true</optional>
+    </dependency>
   </dependencies>
 </project>
diff --git a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
index 15877b03ad93..8fddfe004229 100644
--- a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
+++ b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.io.hadoop;
 
+import com.google.auto.service.AutoService;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
@@ -25,9 +26,14 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CoderProvider;
+import org.apache.beam.sdk.coders.CoderProviderRegistrar;
 import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.DefaultCoder;
+import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Writable;
 
@@ -110,4 +116,54 @@ public int hashCode() {
     return type.hashCode();
   }
 
+  /**
+   * Returns a {@link CoderProvider} which uses the {@link WritableCoder} for Hadoop
+   * {@link Writable writable types}.
+   *
+   * <p>This method is invoked reflectively from {@link DefaultCoder}.
+   */
+  public static CoderProvider getCoderProvider() {
+    return new WritableCoderProvider();
+  }
+
+  /**
+   * A {@link CoderProviderRegistrar} which registers a {@link CoderProvider} which can handle
+   * {@link Writable writable types}.
+   */
+  @AutoService(CoderProviderRegistrar.class)
+  public static class WritableCoderProviderRegistrar implements CoderProviderRegistrar {
+
+    @Override
+    public List<CoderProvider> getCoderProviders() {
+      return Collections.singletonList(getCoderProvider());
+    }
+  }
+
+  /**
+   * A {@link CoderProvider} for Hadoop {@link Writable writable types}.
+   */
+  private static class WritableCoderProvider extends CoderProvider {
+    private static final TypeDescriptor<Writable> WRITABLE_TYPE = new TypeDescriptor<Writable>() {};
+
+    @Override
+    public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
+        List<? extends Coder<?>> componentCoders) throws CannotProvideCoderException {
+      if (!typeDescriptor.isSubtypeOf(WRITABLE_TYPE)) {
+        throw new CannotProvideCoderException(
+            String.format(
+                "Cannot provide %s because %s does not implement the interface %s",
+                WritableCoder.class.getSimpleName(),
+                typeDescriptor,
+                Writable.class.getName()));
+      }
+
+      try {
+        @SuppressWarnings("unchecked")
+        Coder<T> coder = WritableCoder.of((Class) typeDescriptor.getRawType());
+        return coder;
+      } catch (IllegalArgumentException e) {
+        throw new CannotProvideCoderException(e);
+      }
+    }
+  }
 }
diff --git a/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/WritableCoderTest.java b/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/WritableCoderTest.java
index 8127773e03e6..9f2a54dd9700 100644
--- a/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/WritableCoderTest.java
+++ b/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/WritableCoderTest.java
@@ -17,6 +17,10 @@
  */
 package org.apache.beam.sdk.io.hadoop;
 
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.testing.CoderProperties;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.NullWritable;
@@ -42,4 +46,10 @@ public void testNullWritableEncoding() throws Exception {
 
     CoderProperties.coderDecodeEncodeEqual(coder, value);
   }
+
+  @Test
+  public void testAutomaticRegistrationOfCoderProvider() throws Exception {
+    assertThat(CoderRegistry.createDefault().getCoder(NullWritable.class),
+        instanceOf(WritableCoder.class));
+  }
 }
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index e67645529a1f..ba84c2a3484a 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -304,7 +304,7 @@ static <T> NullableCoder<T> inferCoder(
         Class<T> clazz = (Class<T>) parameter;
 
         try {
-          return NullableCoder.of(coderRegistry.getDefaultCoder(clazz));
+          return NullableCoder.of(coderRegistry.getCoder(clazz));
         } catch (CannotProvideCoderException e) {
           throw new RuntimeException(
                   String.format("Unable to automatically infer a Coder for "
diff --git a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/DistinctJava8Test.java b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/DistinctJava8Test.java
index 790f51e06c91..61df23bc6057 100644
--- a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/DistinctJava8Test.java
+++ b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/DistinctJava8Test.java
@@ -18,7 +18,6 @@
 package org.apache.beam.sdk.transforms;
 
 import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertThat;
@@ -89,13 +88,6 @@ public void withLambdaRepresentativeValuesFnNoTypeDescriptorShouldThrow() {
 
     thrown.expect(IllegalStateException.class);
     thrown.expectMessage("Unable to return a default Coder for RemoveRepresentativeDupes");
-    thrown.expectMessage("No Coder has been manually specified");
-    thrown.expectMessage(
-        containsString("Building a Coder using a registered CoderFactory failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the @DefaultCoder annotation failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the fallback CoderProvider failed"));
 
     // Thrown when applying a transform to the internal WithKeys that withRepresentativeValueFn is
     // implemented with
diff --git a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/FilterJava8Test.java b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/FilterJava8Test.java
index a3481e121786..b38250e6147a 100644
--- a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/FilterJava8Test.java
+++ b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/FilterJava8Test.java
@@ -91,7 +91,7 @@ public void testFilterParDoOutputTypeDescriptorRaw() throws Exception {
         .apply(Filter.by(s -> true));
 
     thrown.expect(CannotProvideCoderException.class);
-    pipeline.getCoderRegistry().getDefaultCoder(output.getTypeDescriptor());
+    pipeline.getCoderRegistry().getCoder(output.getTypeDescriptor());
   }
 
   @Test
diff --git a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/PartitionJava8Test.java b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/PartitionJava8Test.java
index 7d977402dd54..94353a5bf7df 100644
--- a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/PartitionJava8Test.java
+++ b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/PartitionJava8Test.java
@@ -69,6 +69,6 @@ public void testPartitionFnOutputTypeDescriptorRaw() throws Exception {
         .apply(Partition.of(1, (element, numPartitions) -> 0));
 
     thrown.expect(CannotProvideCoderException.class);
-    pipeline.getCoderRegistry().getDefaultCoder(output.get(0).getTypeDescriptor());
+    pipeline.getCoderRegistry().getCoder(output.get(0).getTypeDescriptor());
   }
 }
diff --git a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithKeysJava8Test.java b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithKeysJava8Test.java
index 4f0361e3036a..260dd246bd9e 100644
--- a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithKeysJava8Test.java
+++ b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithKeysJava8Test.java
@@ -17,8 +17,6 @@
  */
 package org.apache.beam.sdk.transforms;
 
-import static org.hamcrest.Matchers.containsString;
-
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.ValidatesRunner;
@@ -69,13 +67,6 @@ public void withLambdaAndNoTypeDescriptorShouldThrow() {
 
     thrown.expect(IllegalStateException.class);
     thrown.expectMessage("Unable to return a default Coder for ApplyKeysWithWithKeys");
-    thrown.expectMessage("No Coder has been manually specified");
-    thrown.expectMessage(
-        containsString("Building a Coder using a registered CoderFactory failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the @DefaultCoder annotation failed"));
-    thrown.expectMessage(
-        containsString("Building a Coder from the fallback CoderProvider failed"));
 
     p.run();
   }

From 56a83c73b9b7ab0760c421cf154caff16953500a Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Fri, 5 May 2017 15:09:26 -0700
Subject: [PATCH 090/387] [BEAM-2187] Fix build break in
 SparkRuntimeContextTest

---
 .../beam/runners/spark/translation/SparkRuntimeContextTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
index 372d46f26765..e8f578a54d8e 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkRuntimeContextTest.java
@@ -105,7 +105,7 @@ public void testSerializingPipelineOptionsWithCustomUserType() throws Exception
         .as(JacksonIncompatibleOptions.class);
     options.setRunner(CrashingRunner.class);
     Pipeline p = Pipeline.create(options);
-    SparkRuntimeContext context = new SparkRuntimeContext(p);
+    SparkRuntimeContext context = new SparkRuntimeContext(p, options);
 
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     try (ObjectOutputStream outputStream = new ObjectOutputStream(baos)) {

From bf5167017a86ccb51697c651d7f7d50e4256bc4c Mon Sep 17 00:00:00 2001
From: Davor Bonaci <davor@google.com>
Date: Fri, 5 May 2017 15:41:39 -0700
Subject: [PATCH 091/387] [maven-release-plugin] prepare branch release-2.0.0

---
 examples/java/pom.xml                      | 2 +-
 examples/java8/pom.xml                     | 2 +-
 pom.xml                                    | 4 ++--
 runners/google-cloud-dataflow-java/pom.xml | 2 +-
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml   | 3 +--
 5 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index fb2613514e8c..2f97c505ef9b 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -407,7 +407,7 @@
                 </filter>
               </filters>
               <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
               </transformers>
             </configuration>
           </execution>
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 2180a490144c..c7e5f89cc8cb 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -180,7 +180,7 @@
                 </filter>
               </filters>
               <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
               </transformers>
             </configuration>
           </execution>
diff --git a/pom.xml b/pom.xml
index 15428913d9fa..03fa6c372c17 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,7 +48,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/beam.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/beam.git</developerConnection>
     <url>https://git-wip-us.apache.org/repos/asf?p=beam.git;a=summary</url>
-    <tag>HEAD</tag>
+    <tag>release-2.0.0</tag>
   </scm>
 
   <issueManagement>
@@ -1438,7 +1438,7 @@
                   </relocation>
                 </relocations>
                 <transformers>
-                  <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                  <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
                 </transformers>
               </configuration>
             </execution>
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index c7142fe024b2..13ce1a41b451 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -223,7 +223,7 @@
                 </relocation>
               </relocations>
               <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
               </transformers>
             </configuration>
           </execution>
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
index cc5e49a2c0c6..763c3978f6ed 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
+++ b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
@@ -21,8 +21,7 @@
   child module contains only those tests and overrides the enforcer rules to
   allow 1.8 only behavior without making all of HifIO work only with jdk1.8. -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.beam</groupId>

From 7d93b17068ab8f3865117ad446cec8f30fad8444 Mon Sep 17 00:00:00 2001
From: Davor Bonaci <davor@google.com>
Date: Fri, 5 May 2017 15:42:17 -0700
Subject: [PATCH 092/387] [maven-release-plugin] prepare for next development
 iteration

---
 examples/java/pom.xml                                   | 2 +-
 examples/java8/pom.xml                                  | 2 +-
 examples/pom.xml                                        | 2 +-
 pom.xml                                                 | 4 ++--
 runners/apex/pom.xml                                    | 2 +-
 runners/core-construction-java/pom.xml                  | 2 +-
 runners/core-java/pom.xml                               | 2 +-
 runners/direct-java/pom.xml                             | 2 +-
 runners/flink/pom.xml                                   | 2 +-
 runners/google-cloud-dataflow-java/pom.xml              | 2 +-
 runners/pom.xml                                         | 2 +-
 runners/spark/pom.xml                                   | 2 +-
 sdks/common/fn-api/pom.xml                              | 2 +-
 sdks/common/pom.xml                                     | 2 +-
 sdks/common/runner-api/pom.xml                          | 2 +-
 sdks/java/build-tools/pom.xml                           | 2 +-
 sdks/java/core/pom.xml                                  | 2 +-
 sdks/java/extensions/google-cloud-platform-core/pom.xml | 2 +-
 sdks/java/extensions/jackson/pom.xml                    | 2 +-
 sdks/java/extensions/join-library/pom.xml               | 2 +-
 sdks/java/extensions/pom.xml                            | 2 +-
 sdks/java/extensions/protobuf/pom.xml                   | 2 +-
 sdks/java/extensions/sorter/pom.xml                     | 2 +-
 sdks/java/harness/pom.xml                               | 2 +-
 sdks/java/io/common/pom.xml                             | 2 +-
 sdks/java/io/elasticsearch/pom.xml                      | 2 +-
 sdks/java/io/google-cloud-platform/pom.xml              | 2 +-
 sdks/java/io/hadoop-common/pom.xml                      | 2 +-
 sdks/java/io/hadoop-file-system/pom.xml                 | 2 +-
 sdks/java/io/hadoop/input-format/pom.xml                | 2 +-
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml                | 2 +-
 sdks/java/io/hadoop/pom.xml                             | 2 +-
 sdks/java/io/hbase/pom.xml                              | 2 +-
 sdks/java/io/jdbc/pom.xml                               | 2 +-
 sdks/java/io/jms/pom.xml                                | 2 +-
 sdks/java/io/kafka/pom.xml                              | 2 +-
 sdks/java/io/kinesis/pom.xml                            | 2 +-
 sdks/java/io/mongodb/pom.xml                            | 2 +-
 sdks/java/io/mqtt/pom.xml                               | 2 +-
 sdks/java/io/pom.xml                                    | 2 +-
 sdks/java/io/xml/pom.xml                                | 2 +-
 sdks/java/java8tests/pom.xml                            | 2 +-
 sdks/java/javadoc/pom.xml                               | 2 +-
 sdks/java/maven-archetypes/examples-java8/pom.xml       | 2 +-
 sdks/java/maven-archetypes/examples/pom.xml             | 2 +-
 sdks/java/maven-archetypes/pom.xml                      | 2 +-
 sdks/java/maven-archetypes/starter/pom.xml              | 2 +-
 sdks/java/pom.xml                                       | 2 +-
 sdks/pom.xml                                            | 2 +-
 sdks/python/pom.xml                                     | 2 +-
 50 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 2f97c505ef9b..a6ef5d1ee1c7 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-examples-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index c7e5f89cc8cb..5a94beecfda2 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-examples-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/examples/pom.xml b/examples/pom.xml
index c89e389c706b..a7e61dd2ff1e 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 03fa6c372c17..7103a147c7ab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,7 +34,7 @@
   <url>http://beam.apache.org/</url>
   <inceptionYear>2016</inceptionYear>
 
-  <version>0.7.0-SNAPSHOT</version>
+  <version>2.1.0-SNAPSHOT</version>
 
   <licenses>
     <license>
@@ -48,7 +48,7 @@
     <connection>scm:git:https://git-wip-us.apache.org/repos/asf/beam.git</connection>
     <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/beam.git</developerConnection>
     <url>https://git-wip-us.apache.org/repos/asf?p=beam.git;a=summary</url>
-    <tag>release-2.0.0</tag>
+    <tag>HEAD</tag>
   </scm>
 
   <issueManagement>
diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index 85b15fa688d9..f1a8a6245a4f 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-runners-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/runners/core-construction-java/pom.xml b/runners/core-construction-java/pom.xml
index b38e602bae2a..abf0b6512fec 100644
--- a/runners/core-construction-java/pom.xml
+++ b/runners/core-construction-java/pom.xml
@@ -24,7 +24,7 @@
   <parent>
     <artifactId>beam-runners-parent</artifactId>
     <groupId>org.apache.beam</groupId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index ab3f09ed8857..c3a8d254f4dd 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-runners-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index e76f65d56701..ba3cd3ea04b1 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-runners-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index 41224545cf4f..ff73ec1eb9d0 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-runners-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 13ce1a41b451..09901d53202b 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-runners-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/runners/pom.xml b/runners/pom.xml
index 8f3cabdad123..38aada80aa2d 100644
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index f7200d6ed0cf..94f163dfdc5a 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-runners-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/common/fn-api/pom.xml b/sdks/common/fn-api/pom.xml
index e3a583b1eaaf..ded955911533 100644
--- a/sdks/common/fn-api/pom.xml
+++ b/sdks/common/fn-api/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-common-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/common/pom.xml b/sdks/common/pom.xml
index c90e6f8e6217..c621ed51087b 100644
--- a/sdks/common/pom.xml
+++ b/sdks/common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/common/runner-api/pom.xml b/sdks/common/runner-api/pom.xml
index 199fbb398a7c..3f550bfcbb0f 100644
--- a/sdks/common/runner-api/pom.xml
+++ b/sdks/common/runner-api/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-common-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/build-tools/pom.xml b/sdks/java/build-tools/pom.xml
index 391367ad851d..5a2c498553db 100644
--- a/sdks/java/build-tools/pom.xml
+++ b/sdks/java/build-tools/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index b27fea880efe..a6b89b8fdfb9 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/extensions/google-cloud-platform-core/pom.xml b/sdks/java/extensions/google-cloud-platform-core/pom.xml
index b1101ae99288..a1baea1e3289 100644
--- a/sdks/java/extensions/google-cloud-platform-core/pom.xml
+++ b/sdks/java/extensions/google-cloud-platform-core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-extensions-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/extensions/jackson/pom.xml b/sdks/java/extensions/jackson/pom.xml
index be1391464653..4b09c1176faf 100644
--- a/sdks/java/extensions/jackson/pom.xml
+++ b/sdks/java/extensions/jackson/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-extensions-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/extensions/join-library/pom.xml b/sdks/java/extensions/join-library/pom.xml
index 94f867365381..556ec4086ec9 100644
--- a/sdks/java/extensions/join-library/pom.xml
+++ b/sdks/java/extensions/join-library/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-extensions-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/extensions/pom.xml b/sdks/java/extensions/pom.xml
index 4c60f2056604..3d63626f30bd 100644
--- a/sdks/java/extensions/pom.xml
+++ b/sdks/java/extensions/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/extensions/protobuf/pom.xml b/sdks/java/extensions/protobuf/pom.xml
index c331b0737b4e..ae909abb49b1 100644
--- a/sdks/java/extensions/protobuf/pom.xml
+++ b/sdks/java/extensions/protobuf/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-extensions-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/extensions/sorter/pom.xml b/sdks/java/extensions/sorter/pom.xml
index 5485952c806e..9d25f9d991dd 100644
--- a/sdks/java/extensions/sorter/pom.xml
+++ b/sdks/java/extensions/sorter/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-extensions-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/harness/pom.xml b/sdks/java/harness/pom.xml
index d00dfe4b14ce..3918fd9fd383 100644
--- a/sdks/java/harness/pom.xml
+++ b/sdks/java/harness/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/common/pom.xml b/sdks/java/io/common/pom.xml
index 3f6d79db6ae7..f7525fd87c17 100644
--- a/sdks/java/io/common/pom.xml
+++ b/sdks/java/io/common/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-sdks-java-io-parent</artifactId>
-        <version>0.7.0-SNAPSHOT</version>
+        <version>2.1.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/sdks/java/io/elasticsearch/pom.xml b/sdks/java/io/elasticsearch/pom.xml
index 1cf3c33d83ae..03632cea0007 100644
--- a/sdks/java/io/elasticsearch/pom.xml
+++ b/sdks/java/io/elasticsearch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 9c1286c4119e..6846990c98d0 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/hadoop-common/pom.xml b/sdks/java/io/hadoop-common/pom.xml
index 9054f99a7bdf..8749243e1e76 100644
--- a/sdks/java/io/hadoop-common/pom.xml
+++ b/sdks/java/io/hadoop-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/hadoop-file-system/pom.xml b/sdks/java/io/hadoop-file-system/pom.xml
index 3b392c28b27c..423237b7f7fd 100644
--- a/sdks/java/io/hadoop-file-system/pom.xml
+++ b/sdks/java/io/hadoop-file-system/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/hadoop/input-format/pom.xml b/sdks/java/io/hadoop/input-format/pom.xml
index e211541eb873..06f9f113d1f2 100644
--- a/sdks/java/io/hadoop/input-format/pom.xml
+++ b/sdks/java/io/hadoop/input-format/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-hadoop-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <artifactId>beam-sdks-java-io-hadoop-input-format</artifactId>
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
index 763c3978f6ed..1c63448ed96b 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
+++ b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
@@ -26,7 +26,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-hadoop-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <artifactId>beam-sdks-java-io-hadoop-jdk1.8-tests</artifactId>
diff --git a/sdks/java/io/hadoop/pom.xml b/sdks/java/io/hadoop/pom.xml
index 85761824e70e..a1c7a2e94e37 100644
--- a/sdks/java/io/hadoop/pom.xml
+++ b/sdks/java/io/hadoop/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <packaging>pom</packaging>
diff --git a/sdks/java/io/hbase/pom.xml b/sdks/java/io/hbase/pom.xml
index 0798efc83973..70ca10568a4b 100644
--- a/sdks/java/io/hbase/pom.xml
+++ b/sdks/java/io/hbase/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
index 6212c2f23f15..17c26a058f67 100644
--- a/sdks/java/io/jdbc/pom.xml
+++ b/sdks/java/io/jdbc/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/jms/pom.xml b/sdks/java/io/jms/pom.xml
index dd9909e1da46..58009a102eea 100644
--- a/sdks/java/io/jms/pom.xml
+++ b/sdks/java/io/jms/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/kafka/pom.xml b/sdks/java/io/kafka/pom.xml
index 2ae940d142c7..f6f038513f86 100644
--- a/sdks/java/io/kafka/pom.xml
+++ b/sdks/java/io/kafka/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/kinesis/pom.xml b/sdks/java/io/kinesis/pom.xml
index 25600ff8bed3..cb7064bcdd82 100644
--- a/sdks/java/io/kinesis/pom.xml
+++ b/sdks/java/io/kinesis/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/mongodb/pom.xml b/sdks/java/io/mongodb/pom.xml
index 014e1b6be3bf..912e20cb785b 100644
--- a/sdks/java/io/mongodb/pom.xml
+++ b/sdks/java/io/mongodb/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/mqtt/pom.xml b/sdks/java/io/mqtt/pom.xml
index dcfcaa80ec56..baaf771dcfb2 100644
--- a/sdks/java/io/mqtt/pom.xml
+++ b/sdks/java/io/mqtt/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/pom.xml b/sdks/java/io/pom.xml
index 9480705d16a8..9657612ec7f7 100644
--- a/sdks/java/io/pom.xml
+++ b/sdks/java/io/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/io/xml/pom.xml b/sdks/java/io/xml/pom.xml
index 25d5d0fa9c45..cf7dd3364d5e 100644
--- a/sdks/java/io/xml/pom.xml
+++ b/sdks/java/io/xml/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-io-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/java8tests/pom.xml b/sdks/java/java8tests/pom.xml
index cafae11fdeb6..b90a757c7f44 100644
--- a/sdks/java/java8tests/pom.xml
+++ b/sdks/java/java8tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/javadoc/pom.xml b/sdks/java/javadoc/pom.xml
index b6aa97833c85..54dae3ad7268 100644
--- a/sdks/java/javadoc/pom.xml
+++ b/sdks/java/javadoc/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/maven-archetypes/examples-java8/pom.xml b/sdks/java/maven-archetypes/examples-java8/pom.xml
index 79b4d462267d..b57644d74aae 100644
--- a/sdks/java/maven-archetypes/examples-java8/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/maven-archetypes/examples/pom.xml b/sdks/java/maven-archetypes/examples/pom.xml
index 54cac44cd6ba..c1378cbc385c 100644
--- a/sdks/java/maven-archetypes/examples/pom.xml
+++ b/sdks/java/maven-archetypes/examples/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/maven-archetypes/pom.xml b/sdks/java/maven-archetypes/pom.xml
index 9d39a1e2e949..7bafefc438fc 100644
--- a/sdks/java/maven-archetypes/pom.xml
+++ b/sdks/java/maven-archetypes/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/maven-archetypes/starter/pom.xml b/sdks/java/maven-archetypes/starter/pom.xml
index 0ef81b3855fc..17ed0de67410 100644
--- a/sdks/java/maven-archetypes/starter/pom.xml
+++ b/sdks/java/maven-archetypes/starter/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-java-maven-archetypes-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/java/pom.xml b/sdks/java/pom.xml
index 7ca61090e7b2..250c85aa7460 100644
--- a/sdks/java/pom.xml
+++ b/sdks/java/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/pom.xml b/sdks/pom.xml
index 9ce658589978..ac84471f8315 100644
--- a/sdks/pom.xml
+++ b/sdks/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml
index f9134a507428..129565424e05 100644
--- a/sdks/python/pom.xml
+++ b/sdks/python/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-sdks-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From 5ce3adf9e3f37616828e7ec9083ec743ba64cf12 Mon Sep 17 00:00:00 2001
From: Davor Bonaci <davor@google.com>
Date: Fri, 5 May 2017 15:48:54 -0700
Subject: [PATCH 093/387] Update archetype reference after changing version
 identifier

---
 .../starter/src/test/resources/projects/basic/reference/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
index 1c9847a126c6..bf38cf391b12 100644
--- a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
+++ b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
@@ -25,7 +25,7 @@
   <version>0.1</version>
 
   <properties>
-    <beam.version>0.7.0-SNAPSHOT</beam.version>
+    <beam.version>2.1.0-SNAPSHOT</beam.version>
   </properties>
 
   <repositories>

From 4675f3805e5d0ff93d33a3fcaf48737eb349cfc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elek=2C=20M=C3=A1rton?= <elek@users.noreply.github.com>
Date: Fri, 5 May 2017 16:11:08 -0700
Subject: [PATCH 094/387] [BEAM-806] Use filtering for the version of the
 reference test project

---
 sdks/java/maven-archetypes/pom.xml                        | 8 +++++++-
 .../src/test/resources/projects/basic/reference/pom.xml   | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/sdks/java/maven-archetypes/pom.xml b/sdks/java/maven-archetypes/pom.xml
index 7bafefc438fc..08be5b246f44 100644
--- a/sdks/java/maven-archetypes/pom.xml
+++ b/sdks/java/maven-archetypes/pom.xml
@@ -54,7 +54,13 @@
         </excludes>
       </resource>
     </resources>
-
+    <!-- needed to replace version in the reference projects -->
+    <testResources>
+      <testResource>
+        <directory>src/test/resources</directory>
+        <filtering>true</filtering>
+      </testResource>
+    </testResources>
     <pluginManagement>
       <plugins>
         <plugin>
diff --git a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
index bf38cf391b12..39fefd6ff60a 100644
--- a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
+++ b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
@@ -25,7 +25,7 @@
   <version>0.1</version>
 
   <properties>
-    <beam.version>2.1.0-SNAPSHOT</beam.version>
+    <beam.version>@project.version@</beam.version>
   </properties>
 
   <repositories>

From 1aaca2bf0273287831f673b1b22dd16bfb94878d Mon Sep 17 00:00:00 2001
From: Davor Bonaci <davor@google.com>
Date: Fri, 5 May 2017 16:14:07 -0700
Subject: [PATCH 095/387] PR polish: move testResources section from
 all-archetypes level to starter-archetype level

---
 sdks/java/maven-archetypes/pom.xml         | 9 ++-------
 sdks/java/maven-archetypes/starter/pom.xml | 8 ++++++++
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/sdks/java/maven-archetypes/pom.xml b/sdks/java/maven-archetypes/pom.xml
index 08be5b246f44..0a37c5216d1f 100644
--- a/sdks/java/maven-archetypes/pom.xml
+++ b/sdks/java/maven-archetypes/pom.xml
@@ -54,13 +54,7 @@
         </excludes>
       </resource>
     </resources>
-    <!-- needed to replace version in the reference projects -->
-    <testResources>
-      <testResource>
-        <directory>src/test/resources</directory>
-        <filtering>true</filtering>
-      </testResource>
-    </testResources>
+
     <pluginManagement>
       <plugins>
         <plugin>
@@ -76,6 +70,7 @@
       </plugins>
     </pluginManagement>
   </build>
+
   <profiles>
     <!--
       The builds for the example archetypes use unix scripting, so are enabled only on
diff --git a/sdks/java/maven-archetypes/starter/pom.xml b/sdks/java/maven-archetypes/starter/pom.xml
index 17ed0de67410..06b41c8f0850 100644
--- a/sdks/java/maven-archetypes/starter/pom.xml
+++ b/sdks/java/maven-archetypes/starter/pom.xml
@@ -42,6 +42,14 @@
       </extension>
     </extensions>
 
+    <!-- Needed to replace version in the reference projects. -->
+    <testResources>
+      <testResource>
+        <directory>src/test/resources</directory>
+        <filtering>true</filtering>
+      </testResource>
+    </testResources>
+
     <pluginManagement>
       <plugins>
         <plugin>

From 5fdb40554c7625a6e1ecfc5a8516885d112ae1d0 Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Thu, 4 May 2017 18:30:35 -0700
Subject: [PATCH 096/387] Fix default temp location for DataflowRunner

---
 .../runners/dataflow/options/DataflowPipelineOptions.java     | 4 ++++
 .../apache/beam/sdk/extensions/gcp/options/GcpOptions.java    | 4 ++--
 .../beam/sdk/extensions/gcp/options/GcpOptionsTest.java       | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 11618af9b1fe..4af420b4cae4 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -33,6 +33,8 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.options.Validation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Options that can be used to configure the {@link DataflowRunner}.
@@ -117,10 +119,12 @@ public interface DataflowPipelineOptions
    * Returns a default staging location under {@link GcpOptions#getGcpTempLocation}.
    */
   class StagingLocationFactory implements DefaultValueFactory<String> {
+    private static final Logger LOG = LoggerFactory.getLogger(StagingLocationFactory.class);
 
     @Override
     public String create(PipelineOptions options) {
       GcsOptions gcsOptions = options.as(GcsOptions.class);
+      LOG.info("No stagingLocation provided, falling back to gcpTempLocation");
       String gcpTempLocation;
       try {
         gcpTempLocation = gcsOptions.getGcpTempLocation();
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index 126b7952d8d2..a4128e803505 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -278,7 +278,7 @@ static String tryCreateDefaultBucket(
       }
       final String bucketName =
           "dataflow-staging-" + region + "-" + projectNumber;
-      LOG.info("No staging location provided, attempting to use default bucket: {}",
+      LOG.info("No tempLocation specified, attempting to use default bucket: {}",
           bucketName);
       Bucket bucket = new Bucket()
           .setName(bucketName)
@@ -306,7 +306,7 @@ static String tryCreateDefaultBucket(
         throw new RuntimeException(
             "Unable to determine the owner of the default bucket at gs://" + bucketName, e);
       }
-      return "gs://" + bucketName;
+      return "gs://" + bucketName + "/temp/";
     }
 
     /**
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
index 68b3818762b0..3cf698030a0d 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
@@ -220,7 +220,7 @@ public void testCreateBucket() throws Exception {
       when(mockGcsUtil.bucketOwner(any(GcsPath.class))).thenReturn(1L);
 
       String bucket = GcpTempLocationFactory.tryCreateDefaultBucket(options, mockCrmClient);
-      assertEquals("gs://dataflow-staging-us-north1-1", bucket);
+      assertEquals("gs://dataflow-staging-us-north1-1/temp/", bucket);
     }
 
     @Test

From d026b4cd0ed8c5dbccf617e7a934c183f9f9eb5b Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 14:07:44 -0700
Subject: [PATCH 097/387] [BEAM-2181] Upgrade to Bigtable 0.9.6.2

---
 pom.xml                                                  | 2 +-
 .../org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java  | 2 +-
 .../beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java    | 9 ---------
 3 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/pom.xml b/pom.xml
index 7103a147c7ab..367a61f2f966 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,7 +106,7 @@
     <apex.kryo.version>2.24.0</apex.kryo.version>
     <avro.version>1.8.1</avro.version>
     <bigquery.version>v2-rev295-1.22.0</bigquery.version>
-    <bigtable.version>0.9.6</bigtable.version>
+    <bigtable.version>0.9.6.2</bigtable.version>
     <cloudresourcemanager.version>v1-rev6-1.22.0</cloudresourcemanager.version>
     <pubsubgrpc.version>0.1.0</pubsubgrpc.version>
     <clouddebugger.version>v2-rev8-1.22.0</clouddebugger.version>
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 846d07bb78a2..bde7ea500c34 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -67,7 +67,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * A bounded source and sink for Google Cloud Bigtable.
+ * {@link PTransform Transforms} for reading from and writing to Google Cloud Bigtable.
  *
  * <p>For more information about Cloud Bigtable, see the online documentation at
  * <a href="https://cloud.google.com/bigtable/">Google Cloud Bigtable</a>.
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
index 90102c8e0a4c..d1a17feab6e5 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
@@ -74,15 +74,6 @@ public BigtableWriterImpl openForWriting(String tableId) throws IOException {
 
   @Override
   public boolean tableExists(String tableId) throws IOException {
-    if (!BigtableSession.isAlpnProviderEnabled()) {
-      LOG.info(
-          "Skipping existence check for table {} (BigtableOptions {}) because ALPN is not"
-              + " configured.",
-          tableId,
-          options);
-      return true;
-    }
-
     try (BigtableSession session = new BigtableSession(options)) {
       GetTableRequest getTable =
           GetTableRequest.newBuilder()

From e287544c16cc3e22228c75bb8599dcd2678bcbd0 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 5 May 2017 19:15:48 -0700
Subject: [PATCH 098/387] Update python dataflow worker

---
 sdks/python/apache_beam/runners/dataflow/internal/dependency.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index f64f9b2e4dab..782cc44f5c92 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -72,7 +72,7 @@
 
 # Update this version to the next version whenever there is a change that will
 # require changes to the execution environment.
-BEAM_CONTAINER_VERSION = 'beam-0.7.0-20170316'
+BEAM_CONTAINER_VERSION = 'beam-0.7.0-20170505'
 
 # Standard file names used for staging files.
 WORKFLOW_TARBALL_FILE = 'workflow.tar.gz'

From 46c2f935a99350e18e5d50f1a996996760ebc2e3 Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Fri, 5 May 2017 23:13:24 +0300
Subject: [PATCH 099/387] [BEAM-1672] Make MetricsContainers accumulable.

---
 .../beam/runners/core/LateDataUtils.java      |   2 +-
 .../apache/beam/sdk/metrics/CounterCell.java  |  27 +-
 .../apache/beam/sdk/metrics/DirtyState.java   |   3 +-
 .../beam/sdk/metrics/DistributionCell.java    |  16 +-
 .../apache/beam/sdk/metrics/GaugeCell.java    |  20 +-
 .../apache/beam/sdk/metrics/MetricCell.java   |  14 +-
 .../org/apache/beam/sdk/metrics/Metrics.java  |   2 +-
 .../beam/sdk/metrics/MetricsContainer.java    |  29 +-
 .../sdk/metrics/MetricsContainerStepMap.java  | 487 ++++++++++++++++++
 .../apache/beam/sdk/metrics/MetricsMap.java   |   5 +-
 .../beam/sdk/metrics/CounterCellTest.java     |   6 +-
 .../metrics/MetricsContainerStepMapTest.java  | 258 ++++++++++
 .../sdk/metrics/MetricsContainerTest.java     |  14 +-
 13 files changed, 846 insertions(+), 37 deletions(-)
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainerStepMap.java
 create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerStepMapTest.java

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
index c45387bacd86..f7c0d31374ea 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
@@ -71,7 +71,7 @@ public boolean apply(@Nullable WindowedValue<V> input) {
                         .isBefore(timerInternals.currentInputWatermarkTime());
                 if (expired) {
                   // The element is too late for this window.
-                  droppedDueToLateness.inc();
+                  droppedDueToLateness.update(1L);
                   WindowTracing.debug(
                       "GroupAlsoByWindow: Dropping element at {} for key: {}; "
                           + "window: {} since it is too far behind inputWatermark: {}",
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
index 7ab5ebcb8e03..4b8548f7da22 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
@@ -30,7 +30,7 @@
  * indirection.
  */
 @Experimental(Kind.METRICS)
-public class CounterCell implements MetricCell<Long> {
+public class CounterCell implements MetricCell<Counter, Long> {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicLong value = new AtomicLong();
@@ -41,12 +41,25 @@ public class CounterCell implements MetricCell<Long> {
    */
   CounterCell() {}
 
-  /** Increment the counter by the given amount. */
-  private void add(long n) {
+  /**
+   * Increment the counter by the given amount.
+   * @param n value to increment by. Can be negative to decrement.
+   */
+  public void update(long n) {
     value.addAndGet(n);
     dirty.afterModification();
   }
 
+  @Override
+  public void update(Long n) {
+    throw new UnsupportedOperationException("CounterCell.update(Long n) should not be used"
+    + " as it performs unnecessary boxing/unboxing. Use CounterCell.update(long n) instead.");
+  }
+
+  @Override public void update(MetricCell<Counter, Long> other) {
+    update((long) other.getCumulative());
+  }
+
   @Override
   public DirtyState getDirty() {
     return dirty;
@@ -56,12 +69,4 @@ public DirtyState getDirty() {
   public Long getCumulative() {
     return value.get();
   }
-
-  public void inc() {
-    add(1);
-  }
-
-  public void inc(long n) {
-    add(n);
-  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DirtyState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DirtyState.java
index 6706be8ef9ca..4e0c15cb2d67 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DirtyState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DirtyState.java
@@ -18,6 +18,7 @@
 
 package org.apache.beam.sdk.metrics;
 
+import java.io.Serializable;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -41,7 +42,7 @@
  * completed.
  */
 @Experimental(Kind.METRICS)
-class DirtyState {
+class DirtyState implements Serializable {
   private enum State {
     /** Indicates that there have been changes to the MetricCell since last commit. */
     DIRTY,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
index 0f3f6a458117..93a3649d19a5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
@@ -30,7 +30,7 @@
  * of indirection.
  */
 @Experimental(Kind.METRICS)
-public class DistributionCell implements MetricCell<DistributionData> {
+public class DistributionCell implements MetricCell<Distribution, DistributionData> {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicReference<DistributionData> value =
@@ -42,15 +42,25 @@ public class DistributionCell implements MetricCell<DistributionData> {
    */
   DistributionCell() {}
 
-  /** Increment the counter by the given amount. */
+  /** Increment the distribution by the given amount. */
   public void update(long n) {
+    update(DistributionData.singleton(n));
+  }
+
+  @Override
+  public void update(DistributionData data) {
     DistributionData original;
     do {
       original = value.get();
-    } while (!value.compareAndSet(original, original.combine(DistributionData.singleton(n))));
+    } while (!value.compareAndSet(original, original.combine(data)));
     dirty.afterModification();
   }
 
+  @Override
+  public void update(MetricCell<Distribution, DistributionData> other) {
+    update(other.getCumulative());
+  }
+
   @Override
   public DirtyState getDirty() {
     return dirty;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeCell.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeCell.java
index 6f8e88050afa..0cdd56891718 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeCell.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeCell.java
@@ -29,17 +29,33 @@
  * of indirection.
  */
 @Experimental(Experimental.Kind.METRICS)
-public class GaugeCell implements MetricCell<GaugeData> {
+public class GaugeCell implements MetricCell<Gauge, GaugeData> {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicReference<GaugeData> gaugeValue = new AtomicReference<>(GaugeData.empty());
 
+  /** Set the gauge to the given value. */
   public void set(long value) {
+    update(GaugeData.create(value));
+  }
+
+  @Override
+  public void update(GaugeData data) {
+    GaugeData original;
+    do {
+      original = gaugeValue.get();
+    } while (!gaugeValue.compareAndSet(original, original.combine(data)));
+    dirty.afterModification();
+  }
+
+  @Override
+  public void update(MetricCell<Gauge, GaugeData> other) {
     GaugeData original;
     do {
       original = gaugeValue.get();
-    } while (!gaugeValue.compareAndSet(original, original.combine(GaugeData.create(value))));
+    } while (!gaugeValue.compareAndSet(original, original.combine(other.getCumulative())));
     dirty.afterModification();
+    update(other.getCumulative());
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java
index 82e30cba3a78..403cac208319 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.metrics;
 
+import java.io.Serializable;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
@@ -24,10 +25,21 @@
  * A {@link MetricCell} is used for accumulating in-memory changes to a metric. It represents a
  * specific metric name in a single context.
  *
+ * @param <UserT> The type of the user interface for reporting changes to this cell.
  * @param <DataT> The type of metric data stored (and extracted) from this cell.
  */
 @Experimental(Kind.METRICS)
-public interface MetricCell<DataT> {
+public interface MetricCell<UserT extends Metric, DataT> extends Serializable {
+
+  /**
+   * Update value of this cell.
+   */
+  void update(DataT data);
+
+  /**
+   * Update value of this cell by merging the value of another cell.
+   */
+  void update(MetricCell<UserT, DataT> other);
 
   /**
    * Return the {@link DirtyState} tracking whether this metric cell contains uncommitted changes.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java
index 9286ea9b0097..096d147d1017 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java
@@ -118,7 +118,7 @@ private DelegatingCounter(MetricName name) {
     @Override public void inc(long n) {
       MetricsContainer container = MetricsEnvironment.getCurrentContainer();
       if (container != null) {
-        container.getCounter(name).inc(n);
+        container.getCounter(name).update(n);
       }
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
index fbb0da30a446..48fa35913971 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
@@ -20,6 +20,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.collect.ImmutableList;
+import java.io.Serializable;
 import java.util.Map;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -37,7 +38,7 @@
  * cumulative values/updates.
  */
 @Experimental(Kind.METRICS)
-public class MetricsContainer {
+public class MetricsContainer implements Serializable {
 
   private final String stepName;
 
@@ -96,7 +97,7 @@ public GaugeCell getGauge(MetricName metricName) {
     return gauges.get(metricName);
   }
 
-  private <UpdateT, CellT extends MetricCell<UpdateT>>
+  private <UserT extends Metric, UpdateT, CellT extends MetricCell<UserT, UpdateT>>
   ImmutableList<MetricUpdate<UpdateT>> extractUpdates(
       MetricsMap<MetricName, CellT> cells) {
     ImmutableList.Builder<MetricUpdate<UpdateT>> updates = ImmutableList.builder();
@@ -120,8 +121,8 @@ public MetricUpdates getUpdates() {
         extractUpdates(gauges));
   }
 
-  private void commitUpdates(MetricsMap<MetricName, ? extends MetricCell<?>> cells) {
-    for (MetricCell<?> cell : cells.values()) {
+  private void commitUpdates(MetricsMap<MetricName, ? extends MetricCell<?, ?>> cells) {
+    for (MetricCell<?, ?> cell : cells.values()) {
       cell.getDirty().afterCommit();
     }
   }
@@ -133,9 +134,10 @@ private void commitUpdates(MetricsMap<MetricName, ? extends MetricCell<?>> cells
   public void commitUpdates() {
     commitUpdates(counters);
     commitUpdates(distributions);
+    commitUpdates(gauges);
   }
 
-  private <UpdateT, CellT extends MetricCell<UpdateT>>
+  private <UserT extends Metric, UpdateT, CellT extends MetricCell<UserT, UpdateT>>
   ImmutableList<MetricUpdate<UpdateT>> extractCumulatives(
       MetricsMap<MetricName, CellT> cells) {
     ImmutableList.Builder<MetricUpdate<UpdateT>> updates = ImmutableList.builder();
@@ -156,4 +158,21 @@ public MetricUpdates getCumulative() {
         extractCumulatives(distributions),
         extractCumulatives(gauges));
   }
+
+  /**
+   * Update values of this {@link MetricsContainer} by merging the value of another cell.
+   */
+  public void update(MetricsContainer other) {
+    updateCells(counters, other.counters);
+    updateCells(distributions, other.distributions);
+    updateCells(gauges, other.gauges);
+  }
+
+  private <UserT extends Metric, DataT, CellT extends MetricCell<UserT, DataT>> void updateCells(
+      MetricsMap<MetricName, CellT> current,
+      MetricsMap<MetricName, CellT> updates) {
+    for (Map.Entry<MetricName, CellT> counter : updates.entries()) {
+      current.get(counter.getKey()).update(counter.getValue());
+    }
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainerStepMap.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainerStepMap.java
new file mode 100644
index 000000000000..d01e970f7283
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainerStepMap.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.metrics;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
+
+/**
+ * Metrics containers by step.
+ *
+ * <p>This class is not thread-safe.</p>
+ */
+public class MetricsContainerStepMap implements Serializable {
+  private Map<String, MetricsContainer> metricsContainers;
+
+  public MetricsContainerStepMap() {
+    this.metricsContainers = new ConcurrentHashMap<>();
+  }
+
+  /**
+   * Returns the container for the given step name.
+   */
+  public MetricsContainer getContainer(String stepName) {
+    if (!metricsContainers.containsKey(stepName)) {
+      metricsContainers.put(stepName, new MetricsContainer(stepName));
+    }
+    return metricsContainers.get(stepName);
+  }
+
+  /**
+   * Update this {@link MetricsContainerStepMap} with all values from given
+   * {@link MetricsContainerStepMap}.
+   */
+  public void updateAll(MetricsContainerStepMap other) {
+    for (Map.Entry<String, MetricsContainer> container : other.metricsContainers.entrySet()) {
+      getContainer(container.getKey()).update(container.getValue());
+    }
+  }
+
+  /**
+   * Update {@link MetricsContainer} for given step in this map with all values from given
+   * {@link MetricsContainer}.
+   */
+  public void update(String step, MetricsContainer container) {
+    getContainer(step).update(container);
+  }
+
+  /**
+   * Returns {@link MetricResults} based on given
+   * {@link MetricsContainerStepMap MetricsContainerStepMaps} of attempted and committed metrics.
+   *
+   * <p>This constructor is intended for runners which support both attempted and committed
+   * metrics.
+   */
+  public static MetricResults asMetricResults(
+      MetricsContainerStepMap attemptedMetricsContainers,
+      MetricsContainerStepMap committedMetricsContainers) {
+    return new MetricsContainerStepMapMetricResults(
+        attemptedMetricsContainers,
+        committedMetricsContainers);
+  }
+
+  /**
+   * Returns {@link MetricResults} based on given {@link MetricsContainerStepMap} of attempted
+   * metrics.
+   *
+   * <p>This constructor is intended for runners which only support `attempted` metrics.
+   * Accessing {@link MetricResult#committed()} in the resulting {@link MetricResults} will result
+   * in an {@link UnsupportedOperationException}.</p>
+   */
+  public static MetricResults asAttemptedOnlyMetricResults(
+      MetricsContainerStepMap attemptedMetricsContainers) {
+    return new MetricsContainerStepMapMetricResults(attemptedMetricsContainers);
+  }
+
+  private Map<String, MetricsContainer> getMetricsContainers() {
+    return metricsContainers;
+  }
+
+  private static class MetricsContainerStepMapMetricResults extends MetricResults {
+    private final Map<MetricKey, AttemptedAndCommitted<Long>> counters = new HashMap<>();
+    private final Map<MetricKey, AttemptedAndCommitted<DistributionData>> distributions =
+        new HashMap<>();
+    private final Map<MetricKey, AttemptedAndCommitted<GaugeData>> gauges = new HashMap<>();
+    private final boolean isCommittedSupported;
+
+    private MetricsContainerStepMapMetricResults(
+        MetricsContainerStepMap attemptedMetricsContainers) {
+      this(attemptedMetricsContainers, new MetricsContainerStepMap(), false);
+    }
+
+    private MetricsContainerStepMapMetricResults(
+        MetricsContainerStepMap attemptedMetricsContainers,
+        MetricsContainerStepMap committedMetricsContainers) {
+      this(attemptedMetricsContainers, committedMetricsContainers, true);
+    }
+
+    private MetricsContainerStepMapMetricResults(
+        MetricsContainerStepMap attemptedMetricsContainers,
+        MetricsContainerStepMap committedMetricsContainers,
+        boolean isCommittedSupported) {
+      for (MetricsContainer container
+          : attemptedMetricsContainers.getMetricsContainers().values()) {
+        MetricUpdates cumulative = container.getCumulative();
+        mergeCounters(counters, cumulative.counterUpdates(), attemptedCounterUpdateFn());
+        mergeDistributions(distributions, cumulative.distributionUpdates(),
+            attemptedDistributionUpdateFn());
+        mergeGauges(gauges, cumulative.gaugeUpdates(), attemptedGaugeUpdateFn());
+      }
+      for (MetricsContainer container
+          : committedMetricsContainers.getMetricsContainers().values()) {
+        MetricUpdates cumulative = container.getCumulative();
+        mergeCounters(counters, cumulative.counterUpdates(), committedCounterUpdateFn());
+        mergeDistributions(distributions, cumulative.distributionUpdates(),
+            committedDistributionUpdateFn());
+        mergeGauges(gauges, cumulative.gaugeUpdates(), committedGaugeUpdateFn());
+      }
+      this.isCommittedSupported = isCommittedSupported;
+    }
+
+    private Function<MetricUpdate<DistributionData>, AttemptedAndCommitted<DistributionData>>
+    attemptedDistributionUpdateFn() {
+      return new Function<MetricUpdate<DistributionData>,
+          AttemptedAndCommitted<DistributionData>>() {
+        @Override
+        public AttemptedAndCommitted<DistributionData> apply(MetricUpdate<DistributionData> input) {
+          MetricKey key = input.getKey();
+          return new AttemptedAndCommitted<>(
+              key,
+              input,
+              MetricUpdate.create(key, DistributionData.EMPTY));
+        }
+      };
+    }
+
+    private Function<MetricUpdate<DistributionData>, AttemptedAndCommitted<DistributionData>>
+    committedDistributionUpdateFn() {
+      return new Function<MetricUpdate<DistributionData>,
+          AttemptedAndCommitted<DistributionData>>() {
+        @Override
+        public AttemptedAndCommitted<DistributionData> apply(MetricUpdate<DistributionData> input) {
+          MetricKey key = input.getKey();
+          return new AttemptedAndCommitted<>(
+              key,
+              MetricUpdate.create(key, DistributionData.EMPTY),
+              input);
+        }
+      };
+    }
+
+    private Function<MetricUpdate<GaugeData>, AttemptedAndCommitted<GaugeData>>
+    attemptedGaugeUpdateFn() {
+      return new Function<MetricUpdate<GaugeData>, AttemptedAndCommitted<GaugeData>>() {
+        @Override
+        public AttemptedAndCommitted<GaugeData> apply(MetricUpdate<GaugeData> input) {
+          MetricKey key = input.getKey();
+          return new AttemptedAndCommitted<>(
+              key,
+              input,
+              MetricUpdate.create(key, GaugeData.empty()));
+        }
+      };
+    }
+
+    private Function<MetricUpdate<GaugeData>, AttemptedAndCommitted<GaugeData>>
+    committedGaugeUpdateFn() {
+      return new Function<MetricUpdate<GaugeData>, AttemptedAndCommitted<GaugeData>>() {
+        @Override
+        public AttemptedAndCommitted<GaugeData> apply(MetricUpdate<GaugeData> input) {
+          MetricKey key = input.getKey();
+          return new AttemptedAndCommitted<>(
+              key,
+              MetricUpdate.create(key, GaugeData.empty()),
+              input);
+        }
+      };
+    }
+
+    private Function<MetricUpdate<Long>, AttemptedAndCommitted<Long>> attemptedCounterUpdateFn() {
+      return new Function<MetricUpdate<Long>, AttemptedAndCommitted<Long>>() {
+        @Override
+        public AttemptedAndCommitted<Long> apply(MetricUpdate<Long> input) {
+          MetricKey key = input.getKey();
+          return new AttemptedAndCommitted<>(
+              key,
+              input,
+              MetricUpdate.create(key, 0L));
+        }
+      };
+    }
+
+    private Function<MetricUpdate<Long>, AttemptedAndCommitted<Long>> committedCounterUpdateFn() {
+      return new Function<MetricUpdate<Long>, AttemptedAndCommitted<Long>>() {
+        @Override
+        public AttemptedAndCommitted<Long> apply(MetricUpdate<Long> input) {
+          MetricKey key = input.getKey();
+          return new AttemptedAndCommitted<>(
+              key,
+              MetricUpdate.create(key, 0L),
+              input);
+        }
+      };
+    }
+
+    @Override
+    public MetricQueryResults queryMetrics(MetricsFilter filter) {
+      return new QueryResults(filter);
+    }
+
+    private class QueryResults implements MetricQueryResults {
+      private final MetricsFilter filter;
+
+      private QueryResults(MetricsFilter filter) {
+        this.filter = filter;
+      }
+
+      @Override
+      public Iterable<MetricResult<Long>> counters() {
+        return
+            FluentIterable
+                .from(counters.values())
+                .filter(matchesFilter(filter))
+                .transform(counterUpdateToResult())
+                .toList();
+      }
+
+      @Override
+      public Iterable<MetricResult<DistributionResult>> distributions() {
+        return
+            FluentIterable
+                .from(distributions.values())
+                .filter(matchesFilter(filter))
+                .transform(distributionUpdateToResult())
+                .toList();
+      }
+
+      @Override
+      public Iterable<MetricResult<GaugeResult>> gauges() {
+        return
+            FluentIterable
+                .from(gauges.values())
+                .filter(matchesFilter(filter))
+                .transform(gaugeUpdateToResult())
+                .toList();
+      }
+
+      private Predicate<AttemptedAndCommitted<?>> matchesFilter(final MetricsFilter filter) {
+        return new Predicate<AttemptedAndCommitted<?>>() {
+          @Override
+          public boolean apply(AttemptedAndCommitted<?> attemptedAndCommitted) {
+            return MetricFiltering.matches(filter, attemptedAndCommitted.getKey());
+          }
+        };
+      }
+    }
+
+    private Function<AttemptedAndCommitted<Long>, MetricResult<Long>> counterUpdateToResult() {
+      return new
+          Function<AttemptedAndCommitted<Long>, MetricResult<Long>>() {
+            @Override
+            public MetricResult<Long>
+            apply(AttemptedAndCommitted<Long> metricResult) {
+              MetricKey key = metricResult.getKey();
+              return new AccumulatedMetricResult<>(
+                  key.metricName(),
+                  key.stepName(),
+                  metricResult.getAttempted().getUpdate(),
+                  isCommittedSupported
+                      ? metricResult.getCommitted().getUpdate()
+                      : null,
+                  isCommittedSupported);
+            }
+          };
+    }
+
+    private Function<AttemptedAndCommitted<DistributionData>, MetricResult<DistributionResult>>
+    distributionUpdateToResult() {
+      return new
+          Function<AttemptedAndCommitted<DistributionData>, MetricResult<DistributionResult>>() {
+            @Override
+            public MetricResult<DistributionResult>
+            apply(AttemptedAndCommitted<DistributionData> metricResult) {
+              MetricKey key = metricResult.getKey();
+              return new AccumulatedMetricResult<>(
+                  key.metricName(),
+                  key.stepName(),
+                  metricResult.getAttempted().getUpdate().extractResult(),
+                  isCommittedSupported
+                      ? metricResult.getCommitted().getUpdate().extractResult()
+                      : null,
+                  isCommittedSupported);
+            }
+          };
+    }
+
+    private Function<AttemptedAndCommitted<GaugeData>, MetricResult<GaugeResult>>
+    gaugeUpdateToResult() {
+      return new
+          Function<AttemptedAndCommitted<GaugeData>, MetricResult<GaugeResult>>() {
+            @Override
+            public MetricResult<GaugeResult>
+            apply(AttemptedAndCommitted<GaugeData> metricResult) {
+              MetricKey key = metricResult.getKey();
+              return new AccumulatedMetricResult<>(
+                  key.metricName(),
+                  key.stepName(),
+                  metricResult.getAttempted().getUpdate().extractResult(),
+                  isCommittedSupported
+                      ? metricResult.getCommitted().getUpdate().extractResult()
+                      : null,
+                  isCommittedSupported);
+            }
+          };
+    }
+
+    @SuppressWarnings("ConstantConditions")
+    private void mergeCounters(
+        Map<MetricKey, AttemptedAndCommitted<Long>> counters,
+        Iterable<MetricUpdate<Long>> updates,
+        Function<MetricUpdate<Long>, AttemptedAndCommitted<Long>> updateToAttemptedAndCommittedFn) {
+      for (MetricUpdate<Long> metricUpdate : updates) {
+        MetricKey key = metricUpdate.getKey();
+        AttemptedAndCommitted<Long> update =
+            updateToAttemptedAndCommittedFn.apply(metricUpdate);
+        if (counters.containsKey(key)) {
+          AttemptedAndCommitted<Long> current = counters.get(key);
+          update = new AttemptedAndCommitted<>(
+              key,
+              MetricUpdate.create(
+                  key,
+                  update.getAttempted().getUpdate() + current.getAttempted().getUpdate()),
+              MetricUpdate.create(
+                  key,
+                  update.getCommitted().getUpdate() + current.getCommitted().getUpdate()));
+        }
+        counters.put(key, update);
+      }
+    }
+
+    @SuppressWarnings("ConstantConditions")
+    private void mergeDistributions(
+        Map<MetricKey, AttemptedAndCommitted<DistributionData>> distributions,
+        Iterable<MetricUpdate<DistributionData>> updates,
+        Function<MetricUpdate<DistributionData>, AttemptedAndCommitted<DistributionData>>
+            updateToAttemptedAndCommittedFn) {
+      for (MetricUpdate<DistributionData> metricUpdate : updates) {
+        MetricKey key = metricUpdate.getKey();
+        AttemptedAndCommitted<DistributionData> update =
+            updateToAttemptedAndCommittedFn.apply(metricUpdate);
+        if (distributions.containsKey(key)) {
+          AttemptedAndCommitted<DistributionData> current = distributions.get(key);
+          update = new AttemptedAndCommitted<>(
+              key,
+              MetricUpdate.create(
+                  key,
+                  update.getAttempted().getUpdate().combine(current.getAttempted().getUpdate())),
+              MetricUpdate.create(
+                  key,
+                  update.getCommitted().getUpdate().combine(current.getCommitted().getUpdate())));
+        }
+        distributions.put(key, update);
+      }
+    }
+
+    @SuppressWarnings("ConstantConditions")
+    private void mergeGauges(
+        Map<MetricKey, AttemptedAndCommitted<GaugeData>> gauges,
+        Iterable<MetricUpdate<GaugeData>> updates,
+        Function<MetricUpdate<GaugeData>, AttemptedAndCommitted<GaugeData>>
+            updateToAttemptedAndCommittedFn) {
+      for (MetricUpdate<GaugeData> metricUpdate : updates) {
+        MetricKey key = metricUpdate.getKey();
+        AttemptedAndCommitted<GaugeData> update =
+            updateToAttemptedAndCommittedFn.apply(metricUpdate);
+        if (gauges.containsKey(key)) {
+          AttemptedAndCommitted<GaugeData> current = gauges.get(key);
+          update = new AttemptedAndCommitted<>(
+              key,
+              MetricUpdate.create(
+                  key,
+                  update.getAttempted().getUpdate().combine(current.getAttempted().getUpdate())),
+              MetricUpdate.create(
+                  key,
+                  update.getCommitted().getUpdate().combine(current.getCommitted().getUpdate())));
+        }
+        gauges.put(key, update);
+      }
+    }
+
+    /**
+     * Accumulated implementation of {@link MetricResult}.
+     */
+    private static class AccumulatedMetricResult<T> implements MetricResult<T> {
+      private final MetricName name;
+      private final String step;
+      private final T attempted;
+      private final T committed;
+      private final boolean isCommittedSupported;
+
+      private AccumulatedMetricResult(
+          MetricName name,
+          String step,
+          T attempted,
+          T committed,
+          boolean isCommittedSupported) {
+        this.name = name;
+        this.step = step;
+        this.attempted = attempted;
+        this.committed = committed;
+        this.isCommittedSupported = isCommittedSupported;
+      }
+
+      @Override
+      public MetricName name() {
+        return name;
+      }
+
+      @Override
+      public String step() {
+        return step;
+      }
+
+      @Override
+      public T committed() {
+        if (!isCommittedSupported) {
+          throw new UnsupportedOperationException("This runner does not currently support committed"
+              + " metrics results. Please use 'attempted' instead.");
+        }
+        return committed;
+      }
+
+      @Override
+      public T attempted() {
+        return attempted;
+      }
+    }
+
+    /**
+     * Attempted and committed {@link MetricUpdate MetricUpdates}.
+     */
+    private static class AttemptedAndCommitted<T> {
+      private final MetricKey key;
+      private final MetricUpdate<T> attempted;
+      private final MetricUpdate<T> committed;
+
+      private AttemptedAndCommitted(MetricKey key, MetricUpdate<T> attempted,
+          MetricUpdate<T> committed) {
+        this.key = key;
+        this.attempted = attempted;
+        this.committed = committed;
+      }
+
+      private MetricKey getKey() {
+        return key;
+      }
+
+      private MetricUpdate<T> getAttempted() {
+        return attempted;
+      }
+
+      private MetricUpdate<T> getCommitted() {
+        return committed;
+      }
+    }
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsMap.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsMap.java
index 5a02106f160b..8c26f1869755 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsMap.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsMap.java
@@ -19,6 +19,7 @@
 
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.Iterables;
+import java.io.Serializable;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -31,10 +32,10 @@
  * in a thread-safe manner.
  */
 @Experimental(Kind.METRICS)
-public class MetricsMap<K, T> {
+public class MetricsMap<K, T> implements Serializable {
 
   /** Interface for creating instances to populate the {@link MetricsMap}. */
-  public interface Factory<K, T> {
+  public interface Factory<K, T> extends Serializable {
     /**
      * Create an instance of {@code T} to use with the given {@code key}.
      *
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/CounterCellTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/CounterCellTest.java
index 408f14544aeb..26554d4bc9a4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/CounterCellTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/CounterCellTest.java
@@ -35,8 +35,8 @@ public class CounterCellTest {
 
   @Test
   public void testDeltaAndCumulative() {
-    cell.inc(5);
-    cell.inc(7);
+    cell.update(5);
+    cell.update(7);
     assertThat(cell.getCumulative(), equalTo(12L));
     assertThat("getCumulative is idempotent", cell.getCumulative(), equalTo(12L));
 
@@ -45,7 +45,7 @@ public void testDeltaAndCumulative() {
     assertThat(cell.getDirty().beforeCommit(), equalTo(false));
     assertThat(cell.getCumulative(), equalTo(12L));
 
-    cell.inc(30);
+    cell.update(30);
     assertThat(cell.getCumulative(), equalTo(42L));
 
     assertThat(cell.getDirty().beforeCommit(), equalTo(true));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerStepMapTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerStepMapTest.java
new file mode 100644
index 000000000000..0428ce1aa84e
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerStepMapTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.metrics;
+
+import static org.apache.beam.sdk.metrics.MetricMatchers.metricsResult;
+import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asMetricResults;
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import java.io.Closeable;
+import java.io.IOException;
+import org.hamcrest.collection.IsIterableWithSize;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Tests for {@link MetricsContainerStepMap}.
+ */
+public class MetricsContainerStepMapTest {
+
+  private static final String NAMESPACE = MetricsContainerStepMapTest.class.getName();
+  private static final String STEP1 = "myStep1";
+  private static final String STEP2 = "myStep2";
+
+  private static final long VALUE = 100;
+
+  private static final Counter counter =
+      Metrics.counter(
+          MetricsContainerStepMapTest.class,
+          "myCounter");
+  private static final Distribution distribution =
+      Metrics.distribution(
+          MetricsContainerStepMapTest.class,
+          "myDistribution");
+  private static final Gauge gauge =
+      Metrics.gauge(
+          MetricsContainerStepMapTest.class,
+          "myGauge");
+
+  private static final MetricsContainer metricsContainer;
+
+  static {
+    metricsContainer = new MetricsContainer(null);
+    try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
+      counter.inc(VALUE);
+      distribution.update(VALUE);
+      distribution.update(VALUE * 2);
+      gauge.set(VALUE);
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
+  @Rule
+  public transient ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  public void testAttemptedAccumulatedMetricResults() {
+    MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
+    attemptedMetrics.update(STEP1, metricsContainer);
+    attemptedMetrics.update(STEP2, metricsContainer);
+    attemptedMetrics.update(STEP2, metricsContainer);
+
+    MetricResults metricResults =
+        asAttemptedOnlyMetricResults(attemptedMetrics);
+
+    MetricQueryResults step1res =
+        metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());
+
+    assertIterableSize(step1res.counters(), 1);
+    assertIterableSize(step1res.distributions(), 1);
+    assertIterableSize(step1res.gauges(), 1);
+
+    assertCounter(step1res, STEP1, VALUE, false);
+    assertDistribution(step1res, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
+        false);
+    assertGauge(step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);
+
+    MetricQueryResults step2res =
+        metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build());
+
+    assertIterableSize(step2res.counters(), 1);
+    assertIterableSize(step2res.distributions(), 1);
+    assertIterableSize(step2res.gauges(), 1);
+
+    assertCounter(step2res, STEP2, VALUE * 2, false);
+    assertDistribution(step2res, STEP2, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
+        false);
+    assertGauge(step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);
+
+    MetricQueryResults allres =
+        metricResults.queryMetrics(MetricsFilter.builder().build());
+
+    assertIterableSize(allres.counters(), 2);
+    assertIterableSize(allres.distributions(), 2);
+    assertIterableSize(allres.gauges(), 2);
+  }
+
+  @Test
+  public void testCounterCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
+    MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
+    attemptedMetrics.update(STEP1, metricsContainer);
+    MetricResults metricResults =
+        asAttemptedOnlyMetricResults(attemptedMetrics);
+
+    MetricQueryResults step1res =
+        metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());
+
+    thrown.expect(UnsupportedOperationException.class);
+    thrown.expectMessage("This runner does not currently support committed metrics results.");
+
+    assertCounter(step1res, STEP1, VALUE, true);
+  }
+
+  @Test
+  public void testDistributionCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
+    MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
+    attemptedMetrics.update(STEP1, metricsContainer);
+    MetricResults metricResults =
+        asAttemptedOnlyMetricResults(attemptedMetrics);
+
+    MetricQueryResults step1res =
+        metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());
+
+    thrown.expect(UnsupportedOperationException.class);
+    thrown.expectMessage("This runner does not currently support committed metrics results.");
+
+    assertDistribution(step1res, STEP1, DistributionResult.ZERO, true);
+  }
+
+  @Test
+  public void testGaugeCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
+    MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
+    attemptedMetrics.update(STEP1, metricsContainer);
+    MetricResults metricResults =
+        asAttemptedOnlyMetricResults(attemptedMetrics);
+
+    MetricQueryResults step1res =
+        metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());
+
+    thrown.expect(UnsupportedOperationException.class);
+    thrown.expectMessage("This runner does not currently support committed metrics results.");
+
+    assertGauge(step1res, STEP1, GaugeResult.empty(), true);
+  }
+
+  @Test
+  public void testAttemptedAndCommittedAccumulatedMetricResults() {
+    MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap();
+    attemptedMetrics.update(STEP1, metricsContainer);
+    attemptedMetrics.update(STEP1, metricsContainer);
+    attemptedMetrics.update(STEP2, metricsContainer);
+    attemptedMetrics.update(STEP2, metricsContainer);
+    attemptedMetrics.update(STEP2, metricsContainer);
+
+    MetricsContainerStepMap committedMetrics = new MetricsContainerStepMap();
+    committedMetrics.update(STEP1, metricsContainer);
+    committedMetrics.update(STEP2, metricsContainer);
+    committedMetrics.update(STEP2, metricsContainer);
+
+    MetricResults metricResults =
+        asMetricResults(attemptedMetrics, committedMetrics);
+
+    MetricQueryResults step1res =
+        metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build());
+
+    assertIterableSize(step1res.counters(), 1);
+    assertIterableSize(step1res.distributions(), 1);
+    assertIterableSize(step1res.gauges(), 1);
+
+    assertCounter(step1res, STEP1, VALUE * 2, false);
+    assertDistribution(step1res, STEP1, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
+        false);
+    assertGauge(step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);
+
+    assertCounter(step1res, STEP1, VALUE, true);
+    assertDistribution(step1res, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
+        true);
+    assertGauge(step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), true);
+
+    MetricQueryResults step2res =
+        metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build());
+
+    assertIterableSize(step2res.counters(), 1);
+    assertIterableSize(step2res.distributions(), 1);
+    assertIterableSize(step2res.gauges(), 1);
+
+    assertCounter(step2res, STEP2, VALUE * 3, false);
+    assertDistribution(step2res, STEP2, DistributionResult.create(VALUE * 9, 6, VALUE, VALUE * 2),
+        false);
+    assertGauge(step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);
+
+    assertCounter(step2res, STEP2, VALUE * 2, true);
+    assertDistribution(step2res, STEP2, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
+        true);
+    assertGauge(step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), true);
+
+    MetricQueryResults allres =
+        metricResults.queryMetrics(MetricsFilter.builder().build());
+
+    assertIterableSize(allres.counters(), 2);
+    assertIterableSize(allres.distributions(), 2);
+    assertIterableSize(allres.gauges(), 2);
+  }
+
+  private <T> void assertIterableSize(Iterable<T> iterable, int size) {
+    assertThat(iterable, IsIterableWithSize.<T>iterableWithSize(size));
+  }
+
+  private void assertCounter(
+      MetricQueryResults metricQueryResults,
+      String step,
+      Long expected,
+      boolean isCommitted) {
+    assertThat(
+        metricQueryResults.counters(),
+        hasItem(metricsResult(NAMESPACE, counter.getName().name(), step, expected, isCommitted)));
+  }
+
+  private void assertDistribution(
+      MetricQueryResults metricQueryResults,
+      String step,
+      DistributionResult expected,
+      boolean isCommitted) {
+    assertThat(
+        metricQueryResults.distributions(),
+        hasItem(metricsResult(NAMESPACE, distribution.getName().name(), step, expected,
+            isCommitted)));
+  }
+
+  private void assertGauge(
+      MetricQueryResults metricQueryResults,
+      String step,
+      GaugeResult expected,
+      boolean isCommitted) {
+    assertThat(
+        metricQueryResults.gauges(),
+        hasItem(metricsResult(NAMESPACE, gauge.getName().name(), step, expected, isCommitted)));
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java
index 58797ce2c505..38c00d30e605 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java
@@ -47,8 +47,8 @@ public void testCounterDeltas() {
     assertThat("After commit no counters should be dirty",
         container.getUpdates().counterUpdates(), emptyIterable());
 
-    c1.inc(5L);
-    c2.inc(4L);
+    c1.update(5L);
+    c2.update(4L);
 
     assertThat(container.getUpdates().counterUpdates(), containsInAnyOrder(
         metricUpdate("name1", 5L),
@@ -63,7 +63,7 @@ public void testCounterDeltas() {
     assertThat("After commit there are no updates",
         container.getUpdates().counterUpdates(), emptyIterable());
 
-    c1.inc(8L);
+    c1.update(8L);
     assertThat(container.getUpdates().counterUpdates(), contains(
         metricUpdate("name1", 13L)));
   }
@@ -73,9 +73,9 @@ public void testCounterCumulatives() {
     MetricsContainer container = new MetricsContainer("step1");
     CounterCell c1 = container.getCounter(MetricName.named("ns", "name1"));
     CounterCell c2 = container.getCounter(MetricName.named("ns", "name2"));
-    c1.inc(2L);
-    c2.inc(4L);
-    c1.inc(3L);
+    c1.update(2L);
+    c2.update(4L);
+    c1.update(3L);
 
     container.getUpdates();
     container.commitUpdates();
@@ -84,7 +84,7 @@ public void testCounterCumulatives() {
         metricUpdate("name1", 5L),
         metricUpdate("name2", 4L)));
 
-    c1.inc(8L);
+    c1.update(8L);
     assertThat(container.getCumulative().counterUpdates(), containsInAnyOrder(
         metricUpdate("name1", 13L),
         metricUpdate("name2", 4L)));

From 009cd6e289c77fd3bc997e2bcffaf49d72012c8e Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Fri, 5 May 2017 23:13:51 +0300
Subject: [PATCH 100/387] [BEAM-1672] Use Accumulable MetricsContainers in
 Spark runner.

---
 .../runners/spark/SparkPipelineResult.java    |   8 +-
 .../beam/runners/spark/io/SourceRDD.java      |   4 +-
 .../spark/io/SparkUnboundedSource.java        |  19 +-
 .../spark/metrics/MetricsAccumulator.java     |  20 +-
 .../metrics/MetricsAccumulatorParam.java      |  20 +-
 .../spark/metrics/SparkBeamMetric.java        |  11 +-
 .../spark/metrics/SparkBeamMetricSource.java  |   2 +-
 .../spark/metrics/SparkMetricResults.java     | 172 -----------------
 .../spark/metrics/SparkMetricsContainer.java  | 174 ------------------
 .../SparkGroupAlsoByWindowViaWindowSet.java   |   4 +-
 .../spark/stateful/StateSpecFunctions.java    |   8 +-
 .../translation/DoFnRunnerWithMetrics.java    |   6 +-
 .../spark/translation/MultiDoFnFunction.java  |   6 +-
 .../translation/TransformTranslator.java      |   4 +-
 .../StreamingTransformTranslator.java         |   4 +-
 15 files changed, 66 insertions(+), 396 deletions(-)
 delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricResults.java
 delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
index 3e94a455def3..3986e337dd71 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
@@ -18,13 +18,15 @@
 
 package org.apache.beam.runners.spark;
 
+import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+
 import java.io.IOException;
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import org.apache.beam.runners.spark.metrics.SparkMetricResults;
+import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
 import org.apache.beam.runners.spark.translation.SparkContextFactory;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
@@ -41,7 +43,6 @@ public abstract class SparkPipelineResult implements PipelineResult {
   protected final Future pipelineExecution;
   protected JavaSparkContext javaSparkContext;
   protected PipelineResult.State state;
-  private final SparkMetricResults metricResults = new SparkMetricResults();
 
   SparkPipelineResult(final Future<?> pipelineExecution, final JavaSparkContext javaSparkContext) {
     this.pipelineExecution = pipelineExecution;
@@ -106,7 +107,8 @@ public State waitUntilFinish(final Duration duration) {
 
   @Override
   public MetricResults metrics() {
-    return metricResults;
+    return asAttemptedOnlyMetricResults(
+        MetricsAccumulator.getInstance().value());
   }
 
   @Override
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
index e294359bcb51..71a19e796d4d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
@@ -26,12 +26,12 @@
 import java.util.Iterator;
 import java.util.List;
 import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.metrics.MetricsContainer;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.spark.Accumulator;
@@ -65,7 +65,7 @@ public static class Bounded<T> extends RDD<WindowedValue<T>> {
     private final SparkRuntimeContext runtimeContext;
     private final int numPartitions;
     private final String stepName;
-    private final Accumulator<SparkMetricsContainer> metricsAccum;
+    private final Accumulator<MetricsContainerStepMap> metricsAccum;
 
     // to satisfy Scala API.
     private static final scala.collection.immutable.Seq<Dependency<?>> NIL =
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
index 0388f6c1f1e8..2a9de4b9846b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
@@ -25,7 +25,6 @@
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.runners.spark.stateful.StateSpecFunctions;
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
 import org.apache.beam.runners.spark.translation.streaming.UnboundedDataset;
@@ -37,6 +36,7 @@
 import org.apache.beam.sdk.metrics.Gauge;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.metrics.MetricsContainer;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -190,7 +190,7 @@ public scala.collection.immutable.List<DStream<?>> dependencies() {
     public scala.Option<RDD<BoxedUnit>> compute(Time validTime) {
       // compute parent.
       scala.Option<RDD<Metadata>> parentRDDOpt = parent.getOrCompute(validTime);
-      final Accumulator<SparkMetricsContainer> metricsAccum = MetricsAccumulator.getInstance();
+      final Accumulator<MetricsContainerStepMap> metricsAccum = MetricsAccumulator.getInstance();
       long count = 0;
       SparkWatermarks sparkWatermark = null;
       Instant globalLowWatermarkForBatch = BoundedWindow.TIMESTAMP_MIN_VALUE;
@@ -211,7 +211,7 @@ public scala.Option<RDD<BoxedUnit>> compute(Time validTime) {
                   ? partitionHighWatermark : globalHighWatermarkForBatch;
           // Update metrics reported in the read
           final Gauge gauge = Metrics.gauge(NAMESPACE, READ_DURATION_MILLIS);
-          final MetricsContainer container = metadata.getMetricsContainer().getContainer(stepName);
+          final MetricsContainer container = metadata.getMetricsContainers().getContainer(stepName);
           try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(container)) {
             final long readDurationMillis = metadata.getReadDurationMillis();
             if (readDurationMillis > maxReadDuration) {
@@ -220,7 +220,7 @@ public scala.Option<RDD<BoxedUnit>> compute(Time validTime) {
           } catch (IOException e) {
             throw new RuntimeException(e);
           }
-          metricsAccum.value().update(metadata.getMetricsContainer());
+          metricsAccum.value().updateAll(metadata.getMetricsContainers());
         }
 
         sparkWatermark =
@@ -260,20 +260,19 @@ public static class Metadata implements Serializable {
     private final Instant lowWatermark;
     private final Instant highWatermark;
     private final long readDurationMillis;
-    private final SparkMetricsContainer metricsContainer;
+    private final MetricsContainerStepMap metricsContainers;
 
     public Metadata(
         long numRecords,
         Instant lowWatermark,
         Instant highWatermark,
         final long readDurationMillis,
-        SparkMetricsContainer metricsContainer) {
+        MetricsContainerStepMap metricsContainer) {
       this.numRecords = numRecords;
       this.readDurationMillis = readDurationMillis;
-      this.metricsContainer = metricsContainer;
+      this.metricsContainers = metricsContainer;
       this.lowWatermark = lowWatermark;
       this.highWatermark = highWatermark;
-      metricsContainer.materialize();
     }
 
     long getNumRecords() {
@@ -292,8 +291,8 @@ public long getReadDurationMillis() {
       return readDurationMillis;
     }
 
-    SparkMetricsContainer getMetricsContainer() {
-      return metricsContainer;
+    MetricsContainerStepMap getMetricsContainers() {
+      return metricsContainers;
     }
   }
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
index 1153db6307b9..1dcfa2f11f0d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
@@ -24,6 +24,7 @@
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.translation.streaming.Checkpoint;
 import org.apache.beam.runners.spark.translation.streaming.Checkpoint.CheckpointDir;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.spark.Accumulator;
@@ -44,7 +45,7 @@ public class MetricsAccumulator {
   private static final String ACCUMULATOR_NAME = "Beam.Metrics";
   private static final String ACCUMULATOR_CHECKPOINT_FILENAME = "metrics";
 
-  private static volatile Accumulator<SparkMetricsContainer> instance = null;
+  private static volatile Accumulator<MetricsContainerStepMap> instance = null;
   private static volatile FileSystem fileSystem;
   private static volatile Path checkpointFilePath;
 
@@ -58,11 +59,13 @@ public static void init(SparkPipelineOptions opts, JavaSparkContext jsc) {
           Optional<CheckpointDir> maybeCheckpointDir =
               opts.isStreaming() ? Optional.of(new CheckpointDir(opts.getCheckpointDir()))
                   : Optional.<CheckpointDir>absent();
-          Accumulator<SparkMetricsContainer> accumulator =
-              jsc.sc().accumulator(new SparkMetricsContainer(), ACCUMULATOR_NAME,
+          Accumulator<MetricsContainerStepMap> accumulator =
+              jsc.sc().accumulator(
+                  new MetricsContainerStepMap(),
+                  ACCUMULATOR_NAME,
                   new MetricsAccumulatorParam());
           if (maybeCheckpointDir.isPresent()) {
-            Optional<SparkMetricsContainer> maybeRecoveredValue =
+            Optional<MetricsContainerStepMap> maybeRecoveredValue =
                 recoverValueFromCheckpoint(jsc, maybeCheckpointDir.get());
             if (maybeRecoveredValue.isPresent()) {
               accumulator.setValue(maybeRecoveredValue.get());
@@ -75,7 +78,7 @@ public static void init(SparkPipelineOptions opts, JavaSparkContext jsc) {
     }
   }
 
-  public static Accumulator<SparkMetricsContainer> getInstance() {
+  public static Accumulator<MetricsContainerStepMap> getInstance() {
     if (instance == null) {
       throw new IllegalStateException("Metrics accumulator has not been instantiated");
     } else {
@@ -83,14 +86,15 @@ public static Accumulator<SparkMetricsContainer> getInstance() {
     }
   }
 
-  private static Optional<SparkMetricsContainer> recoverValueFromCheckpoint(
+  private static Optional<MetricsContainerStepMap> recoverValueFromCheckpoint(
       JavaSparkContext jsc,
       CheckpointDir checkpointDir) {
     try {
       Path beamCheckpointPath = checkpointDir.getBeamCheckpointDir();
       checkpointFilePath = new Path(beamCheckpointPath, ACCUMULATOR_CHECKPOINT_FILENAME);
       fileSystem = checkpointFilePath.getFileSystem(jsc.hadoopConfiguration());
-      SparkMetricsContainer recoveredValue = Checkpoint.readObject(fileSystem, checkpointFilePath);
+      MetricsContainerStepMap recoveredValue =
+          Checkpoint.readObject(fileSystem, checkpointFilePath);
       if (recoveredValue != null) {
         LOG.info("Recovered metrics from checkpoint.");
         return Optional.of(recoveredValue);
@@ -117,7 +121,7 @@ private static void checkpoint() throws IOException {
   }
 
   /**
-   * Spark Listener which checkpoints {@link SparkMetricsContainer} values for fault-tolerance.
+   * Spark Listener which checkpoints {@link MetricsContainerStepMap} values for fault-tolerance.
    */
   public static class AccumulatorCheckpointingSparkListener extends JavaStreamingListener {
     @Override
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java
index 9948c815501d..dee4ebc5f821 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java
@@ -18,25 +18,31 @@
 
 package org.apache.beam.runners.spark.metrics;
 
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.spark.AccumulatorParam;
 
 
 /**
  * Metrics accumulator param.
  */
-class MetricsAccumulatorParam implements AccumulatorParam<SparkMetricsContainer> {
+class MetricsAccumulatorParam implements AccumulatorParam<MetricsContainerStepMap> {
   @Override
-  public SparkMetricsContainer addAccumulator(SparkMetricsContainer c1, SparkMetricsContainer c2) {
-    return c1.update(c2);
+  public MetricsContainerStepMap addAccumulator(
+      MetricsContainerStepMap c1,
+      MetricsContainerStepMap c2) {
+    return addInPlace(c1, c2);
   }
 
   @Override
-  public SparkMetricsContainer addInPlace(SparkMetricsContainer c1, SparkMetricsContainer c2) {
-    return c1.update(c2);
+  public MetricsContainerStepMap addInPlace(
+      MetricsContainerStepMap c1,
+      MetricsContainerStepMap c2) {
+    c1.updateAll(c2);
+    return c1;
   }
 
   @Override
-  public SparkMetricsContainer zero(SparkMetricsContainer initialValue) {
-    return new SparkMetricsContainer();
+  public MetricsContainerStepMap zero(MetricsContainerStepMap initialValue) {
+    return new MetricsContainerStepMap();
   }
 }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java
index 2d445a94e4eb..e4bd5989eb76 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java
@@ -18,6 +18,8 @@
 
 package org.apache.beam.runners.spark.metrics;
 
+import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+
 import com.codahale.metrics.Metric;
 import com.google.common.annotations.VisibleForTesting;
 import java.util.HashMap;
@@ -27,20 +29,23 @@
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
 import org.apache.beam.sdk.metrics.MetricResult;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsFilter;
 
 
 /**
- * An adapter between the {@link SparkMetricsContainer} and Codahale's {@link Metric} interface.
+ * An adapter between the {@link MetricsContainerStepMap} and Codahale's {@link Metric} interface.
  */
 class SparkBeamMetric implements Metric {
   private static final String ILLEGAL_CHARACTERS = "[^A-Za-z0-9\\._-]";
   private static final String ILLEGAL_CHARACTERS_AND_PERIOD = "[^A-Za-z0-9_-]";
 
-  private final SparkMetricResults metricResults = new SparkMetricResults();
-
   Map<String, ?> renderAll() {
     Map<String, Object> metrics = new HashMap<>();
+    MetricResults metricResults =
+        asAttemptedOnlyMetricResults(
+            MetricsAccumulator.getInstance().value());
     MetricQueryResults metricQueryResults =
         metricResults.queryMetrics(MetricsFilter.builder().build());
     for (MetricResult<Long> metricResult : metricQueryResults.counters()) {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java
index 5c6fc241265c..03128d70d775 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetricSource.java
@@ -24,7 +24,7 @@
 
 /**
  * A Spark {@link Source} that is tailored to expose a {@link SparkBeamMetric},
- * wrapping an underlying {@link SparkMetricsContainer} instance.
+ * wrapping an underlying {@link org.apache.beam.sdk.metrics.MetricResults} instance.
  */
 public class SparkBeamMetricSource implements Source {
   private static final String METRIC_NAME = "Metrics";
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricResults.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricResults.java
deleted file mode 100644
index faf4c526045b..000000000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricResults.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.metrics;
-
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.collect.FluentIterable;
-import org.apache.beam.sdk.metrics.DistributionData;
-import org.apache.beam.sdk.metrics.DistributionResult;
-import org.apache.beam.sdk.metrics.GaugeData;
-import org.apache.beam.sdk.metrics.GaugeResult;
-import org.apache.beam.sdk.metrics.MetricFiltering;
-import org.apache.beam.sdk.metrics.MetricKey;
-import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricQueryResults;
-import org.apache.beam.sdk.metrics.MetricResult;
-import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
-import org.apache.beam.sdk.metrics.MetricsFilter;
-
-
-/**
- * Implementation of {@link MetricResults} for the Spark Runner.
- */
-public class SparkMetricResults extends MetricResults {
-
-  @Override
-  public MetricQueryResults queryMetrics(MetricsFilter filter) {
-    return new SparkMetricQueryResults(filter);
-  }
-
-  private static class SparkMetricQueryResults implements MetricQueryResults {
-    private final MetricsFilter filter;
-
-    SparkMetricQueryResults(MetricsFilter filter) {
-      this.filter = filter;
-    }
-
-    @Override
-    public Iterable<MetricResult<Long>> counters() {
-      return
-          FluentIterable
-              .from(SparkMetricsContainer.getCounters())
-              .filter(matchesFilter(filter))
-              .transform(TO_COUNTER_RESULT)
-              .toList();
-    }
-
-    @Override
-    public Iterable<MetricResult<DistributionResult>> distributions() {
-      return
-          FluentIterable
-              .from(SparkMetricsContainer.getDistributions())
-              .filter(matchesFilter(filter))
-              .transform(TO_DISTRIBUTION_RESULT)
-              .toList();
-    }
-
-    @Override
-    public Iterable<MetricResult<GaugeResult>> gauges() {
-      return
-          FluentIterable
-              .from(SparkMetricsContainer.getGauges())
-              .filter(matchesFilter(filter))
-              .transform(TO_GAUGE_RESULT)
-              .toList();
-    }
-
-    private Predicate<MetricUpdate<?>> matchesFilter(final MetricsFilter filter) {
-      return new Predicate<MetricUpdate<?>>() {
-        @Override
-        public boolean apply(MetricUpdate<?> metricResult) {
-          return MetricFiltering.matches(filter, metricResult.getKey());
-        }
-      };
-    }
-  }
-
-  private static final Function<MetricUpdate<DistributionData>, MetricResult<DistributionResult>>
-      TO_DISTRIBUTION_RESULT =
-      new Function<MetricUpdate<DistributionData>, MetricResult<DistributionResult>>() {
-        @Override
-        public MetricResult<DistributionResult> apply(MetricUpdate<DistributionData> metricResult) {
-          if (metricResult != null) {
-            MetricKey key = metricResult.getKey();
-            return new SparkMetricResult<>(key.metricName(), key.stepName(),
-                metricResult.getUpdate().extractResult());
-          } else {
-            return null;
-          }
-        }
-      };
-
-  private static final Function<MetricUpdate<Long>, MetricResult<Long>>
-      TO_COUNTER_RESULT =
-      new Function<MetricUpdate<Long>, MetricResult<Long>>() {
-        @Override
-        public MetricResult<Long> apply(MetricUpdate<Long> metricResult) {
-          if (metricResult != null) {
-            MetricKey key = metricResult.getKey();
-            return new SparkMetricResult<>(key.metricName(), key.stepName(),
-                metricResult.getUpdate());
-          } else {
-            return null;
-          }
-        }
-      };
-
-  private static final Function<MetricUpdate<GaugeData>, MetricResult<GaugeResult>>
-      TO_GAUGE_RESULT =
-      new Function<MetricUpdate<GaugeData>, MetricResult<GaugeResult>>() {
-        @Override
-        public MetricResult<GaugeResult> apply(MetricUpdate<GaugeData> metricResult) {
-          if (metricResult != null) {
-            MetricKey key = metricResult.getKey();
-            return new SparkMetricResult<>(key.metricName(), key.stepName(),
-                metricResult.getUpdate().extractResult());
-          } else {
-            return null;
-          }
-        }
-      };
-
-  private static class SparkMetricResult<T> implements MetricResult<T> {
-    private final MetricName name;
-    private final String step;
-    private final T result;
-
-    SparkMetricResult(MetricName name, String step, T result) {
-      this.name = name;
-      this.step = step;
-      this.result = result;
-    }
-
-    @Override
-    public MetricName name() {
-      return name;
-    }
-
-    @Override
-    public String step() {
-      return step;
-    }
-
-    @Override
-    public T committed() {
-      throw new UnsupportedOperationException("Spark runner does not currently support committed"
-          + " metrics results. Please use 'attempted' instead.");
-    }
-
-    @Override
-    public T attempted() {
-      return result;
-    }
-  }
-}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java
deleted file mode 100644
index 9e94c14f6865..000000000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.metrics;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import org.apache.beam.sdk.metrics.DistributionData;
-import org.apache.beam.sdk.metrics.GaugeData;
-import org.apache.beam.sdk.metrics.MetricKey;
-import org.apache.beam.sdk.metrics.MetricUpdates;
-import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
-import org.apache.beam.sdk.metrics.MetricsContainer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Spark accumulator value which holds all {@link MetricsContainer}s, aggregates and merges them.
- */
-public class SparkMetricsContainer implements Serializable {
-  private static final Logger LOG = LoggerFactory.getLogger(SparkMetricsContainer.class);
-
-  private transient volatile LoadingCache<String, MetricsContainer> metricsContainers;
-
-  private final Map<MetricKey, MetricUpdate<Long>> counters = new HashMap<>();
-  private final Map<MetricKey, MetricUpdate<DistributionData>> distributions = new HashMap<>();
-  private final Map<MetricKey, MetricUpdate<GaugeData>> gauges = new HashMap<>();
-
-  public MetricsContainer getContainer(String stepName) {
-    if (metricsContainers == null) {
-      synchronized (this) {
-        if (metricsContainers == null) {
-          initializeMetricsContainers();
-        }
-      }
-    }
-    try {
-      return metricsContainers.get(stepName);
-    } catch (ExecutionException e) {
-      LOG.error("Error while creating metrics container", e);
-      return null;
-    }
-  }
-
-  static Collection<MetricUpdate<Long>> getCounters() {
-    SparkMetricsContainer sparkMetricsContainer = getInstance();
-    sparkMetricsContainer.materialize();
-    return sparkMetricsContainer.counters.values();
-  }
-
-  static Collection<MetricUpdate<DistributionData>> getDistributions() {
-    SparkMetricsContainer sparkMetricsContainer = getInstance();
-    sparkMetricsContainer.materialize();
-    return sparkMetricsContainer.distributions.values();
-  }
-
-  static Collection<MetricUpdate<GaugeData>> getGauges() {
-    return getInstance().gauges.values();
-  }
-
-  public SparkMetricsContainer update(SparkMetricsContainer other) {
-    other.materialize();
-    this.updateCounters(other.counters.values());
-    this.updateDistributions(other.distributions.values());
-    this.updateGauges(other.gauges.values());
-    return this;
-  }
-
-  private static SparkMetricsContainer getInstance() {
-    return MetricsAccumulator.getInstance().value();
-  }
-
-  private void writeObject(ObjectOutputStream out) throws IOException {
-    // Since MetricsContainer instances are not serializable, materialize a serializable map of
-    // MetricsAggregators relating to the same metrics. This is done here, when Spark serializes
-    // the SparkMetricsContainer accumulator before sending results back to the driver at a point in
-    // time where all the metrics updates have already been made to the MetricsContainers.
-    materialize();
-    out.defaultWriteObject();
-  }
-
-  /**
-   * Materialize metrics. Must be called to enable this instance's data to be serialized correctly.
-   * This method is idempotent.
-   */
-  public void materialize() {
-    // Nullifying metricsContainers makes this method idempotent.
-    if (metricsContainers != null) {
-      for (MetricsContainer container : metricsContainers.asMap().values()) {
-        MetricUpdates cumulative = container.getCumulative();
-        this.updateCounters(cumulative.counterUpdates());
-        this.updateDistributions(cumulative.distributionUpdates());
-        this.updateGauges(cumulative.gaugeUpdates());
-      }
-      metricsContainers = null;
-    }
-  }
-
-  private void updateCounters(Iterable<MetricUpdate<Long>> updates) {
-    for (MetricUpdate<Long> update : updates) {
-      MetricKey key = update.getKey();
-      MetricUpdate<Long> current = counters.get(key);
-      counters.put(key, current != null
-          ? MetricUpdate.create(key, current.getUpdate() + update.getUpdate()) : update);
-    }
-  }
-
-  private void updateDistributions(Iterable<MetricUpdate<DistributionData>> updates) {
-    for (MetricUpdate<DistributionData> update : updates) {
-      MetricKey key = update.getKey();
-      MetricUpdate<DistributionData> current = distributions.get(key);
-      distributions.put(key, current != null
-          ? MetricUpdate.create(key, current.getUpdate().combine(update.getUpdate())) : update);
-    }
-  }
-
-  private void updateGauges(Iterable<MetricUpdate<GaugeData>> updates) {
-    for (MetricUpdate<GaugeData> update : updates) {
-      MetricKey key = update.getKey();
-      MetricUpdate<GaugeData> current = gauges.get(key);
-      gauges.put(
-          key,
-          current != null
-              ? MetricUpdate.create(key, current.getUpdate().combine(update.getUpdate()))
-              : update);
-    }
-  }
-
-  private static class MetricsContainerCacheLoader extends CacheLoader<String, MetricsContainer> {
-    @SuppressWarnings("NullableProblems")
-    @Override
-    public MetricsContainer load(String stepName) throws Exception {
-      return new MetricsContainer(stepName);
-    }
-  }
-
-  private void initializeMetricsContainers() {
-    metricsContainers = CacheBuilder.<String, SparkMetricsContainer>newBuilder()
-        .build(new MetricsContainerCacheLoader());
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder();
-    for (Map.Entry<String, ?> metric : new SparkBeamMetric().renderAll().entrySet()) {
-      sb.append(metric.getKey()).append(": ").append(metric.getValue()).append(" ");
-    }
-    return sb.toString();
-  }
-}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 9bc876018d45..37d9635ff5e7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -321,12 +321,12 @@ public JavaPairRDD<ByteArray, byte[]> call(
         long lateDropped = droppedDueToLateness.getCumulative();
         if (lateDropped > 0) {
           LOG.info(String.format("Dropped %d elements due to lateness.", lateDropped));
-          droppedDueToLateness.inc(-droppedDueToLateness.getCumulative());
+          droppedDueToLateness.update(-droppedDueToLateness.getCumulative());
         }
         long closedWindowDropped = droppedDueToClosedWindow.getCumulative();
         if (closedWindowDropped > 0) {
           LOG.info(String.format("Dropped %d elements due to closed window.", closedWindowDropped));
-          droppedDueToClosedWindow.inc(-droppedDueToClosedWindow.getCumulative());
+          droppedDueToClosedWindow.update(-droppedDueToClosedWindow.getCumulative());
         }
 
         return scala.collection.JavaConversions.asScalaIterator(outIter);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
index d8d52c425ecb..17a3c7344bb0 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
@@ -31,12 +31,12 @@
 import org.apache.beam.runners.spark.io.EmptyCheckpointMark;
 import org.apache.beam.runners.spark.io.MicrobatchSource;
 import org.apache.beam.runners.spark.io.SparkUnboundedSource.Metadata;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.metrics.MetricsContainer;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -110,8 +110,8 @@ public Tuple2<Iterable<byte[]>, Metadata> apply(
           scala.Option<CheckpointMarkT> startCheckpointMark,
           State<Tuple2<byte[], Instant>> state) {
 
-        SparkMetricsContainer sparkMetricsContainer = new SparkMetricsContainer();
-        MetricsContainer metricsContainer = sparkMetricsContainer.getContainer(stepName);
+        MetricsContainerStepMap metricsContainers = new MetricsContainerStepMap();
+        MetricsContainer metricsContainer = metricsContainers.getContainer(stepName);
 
         // Add metrics container to the scope of org.apache.beam.sdk.io.Source.Reader methods
         // since they may report metrics.
@@ -214,7 +214,7 @@ public Tuple2<Iterable<byte[]>, Metadata> apply(
                 lowWatermark,
                 highWatermark,
                 readDurationMillis,
-                sparkMetricsContainer));
+                metricsContainers));
 
         } catch (IOException e) {
           throw new RuntimeException(e);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
index d74b25373387..8349b09fab16 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
@@ -21,8 +21,8 @@
 import java.io.Closeable;
 import java.io.IOException;
 import org.apache.beam.runners.core.DoFnRunner;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsContainer;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -37,12 +37,12 @@
 class DoFnRunnerWithMetrics<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
   private final DoFnRunner<InputT, OutputT> delegate;
   private final String stepName;
-  private final Accumulator<SparkMetricsContainer> metricsAccum;
+  private final Accumulator<MetricsContainerStepMap> metricsAccum;
 
   DoFnRunnerWithMetrics(
       String stepName,
       DoFnRunner<InputT, OutputT> delegate,
-      Accumulator<SparkMetricsContainer> metricsAccum) {
+      Accumulator<MetricsContainerStepMap> metricsAccum) {
     this.delegate = delegate;
     this.stepName = stepName;
     this.metricsAccum = metricsAccum;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
index 9bfd2fa3d573..ecf96b62f5a3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
@@ -28,9 +28,9 @@
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.runners.core.DoFnRunners;
 import org.apache.beam.runners.spark.aggregators.NamedAggregators;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.runners.spark.util.SideInputBroadcast;
 import org.apache.beam.runners.spark.util.SparkSideInputReader;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
@@ -52,7 +52,7 @@ public class MultiDoFnFunction<InputT, OutputT>
     implements PairFlatMapFunction<Iterator<WindowedValue<InputT>>, TupleTag<?>, WindowedValue<?>> {
 
   private final Accumulator<NamedAggregators> aggAccum;
-  private final Accumulator<SparkMetricsContainer> metricsAccum;
+  private final Accumulator<MetricsContainerStepMap> metricsAccum;
   private final String stepName;
   private final DoFn<InputT, OutputT> doFn;
   private final SparkRuntimeContext runtimeContext;
@@ -71,7 +71,7 @@ public class MultiDoFnFunction<InputT, OutputT>
    */
   public MultiDoFnFunction(
       Accumulator<NamedAggregators> aggAccum,
-      Accumulator<SparkMetricsContainer> metricsAccum,
+      Accumulator<MetricsContainerStepMap> metricsAccum,
       String stepName,
       DoFn<InputT, OutputT> doFn,
       SparkRuntimeContext runtimeContext,
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index 8a8e24677759..acbac32840d6 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -35,13 +35,13 @@
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.runners.spark.io.SourceRDD;
 import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.runners.spark.util.SideInputBroadcast;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.Create;
@@ -359,7 +359,7 @@ public void evaluate(
         WindowingStrategy<?, ?> windowingStrategy =
             context.getInput(transform).getWindowingStrategy();
         Accumulator<NamedAggregators> aggAccum = AggregatorsAccumulator.getInstance();
-        Accumulator<SparkMetricsContainer> metricsAccum = MetricsAccumulator.getInstance();
+        Accumulator<MetricsContainerStepMap> metricsAccum = MetricsAccumulator.getInstance();
         JavaPairRDD<TupleTag<?>, WindowedValue<?>> all =
             inRDD.mapPartitionsToPair(
                 new MultiDoFnFunction<>(
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index 2c4a7476d0ea..f736e5305b3b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -39,7 +39,6 @@
 import org.apache.beam.runners.spark.io.CreateStream;
 import org.apache.beam.runners.spark.io.SparkUnboundedSource;
 import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.runners.spark.stateful.SparkGroupAlsoByWindowViaWindowSet;
 import org.apache.beam.runners.spark.translation.BoundedDataset;
 import org.apache.beam.runners.spark.translation.Dataset;
@@ -59,6 +58,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -395,7 +395,7 @@ public JavaPairRDD<TupleTag<?>, WindowedValue<?>> call(
                       JavaRDD<WindowedValue<InputT>> rdd) throws Exception {
                     final Accumulator<NamedAggregators> aggAccum =
                         AggregatorsAccumulator.getInstance();
-                    final Accumulator<SparkMetricsContainer> metricsAccum =
+                    final Accumulator<MetricsContainerStepMap> metricsAccum =
                         MetricsAccumulator.getInstance();
                     final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>>
                         sideInputs =

From 8c2da9ad1b8c195757f97feccdbcabcad735c407 Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Fri, 5 May 2017 23:14:01 +0300
Subject: [PATCH 101/387] [BEAM-1672] Use Accumulable MetricsContainers in
 Flink runner.

---
 .../beam/runners/flink/FlinkRunnerResult.java |   8 +-
 .../metrics/DoFnRunnerWithMetricsUpdate.java  |  12 +-
 .../flink/metrics/FlinkMetricContainer.java   | 273 +++++-------------
 .../flink/metrics/FlinkMetricResults.java     | 146 ----------
 .../flink/metrics/MetricsAccumulator.java     |  60 ++++
 .../flink/metrics/ReaderInvocationUtil.java   |   7 +-
 .../wrappers/SourceInputFormat.java           |   8 +-
 .../streaming/io/BoundedSourceWrapper.java    |   8 +-
 .../streaming/io/UnboundedSourceWrapper.java  |   9 +-
 9 files changed, 174 insertions(+), 357 deletions(-)
 delete mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricResults.java
 create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
index 90dc79b3f376..038895ad267d 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
@@ -17,12 +17,15 @@
  */
 package org.apache.beam.runners.flink;
 
+import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
-import org.apache.beam.runners.flink.metrics.FlinkMetricResults;
+import org.apache.beam.runners.flink.metrics.FlinkMetricContainer;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.joda.time.Duration;
 
 /**
@@ -72,6 +75,7 @@ public State waitUntilFinish(Duration duration) {
 
   @Override
   public MetricResults metrics() {
-    return new FlinkMetricResults(accumulators);
+    return asAttemptedOnlyMetricResults(
+        (MetricsContainerStepMap) accumulators.get(FlinkMetricContainer.ACCUMULATOR_NAME));
   }
 }
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
index dae91fe007fe..40191d26b0a2 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
@@ -34,6 +34,7 @@
  */
 public class DoFnRunnerWithMetricsUpdate<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
 
+  private final String stepName;
   private final FlinkMetricContainer container;
   private final DoFnRunner<InputT, OutputT> delegate;
 
@@ -41,14 +42,15 @@ public DoFnRunnerWithMetricsUpdate(
       String stepName,
       DoFnRunner<InputT, OutputT> delegate,
       RuntimeContext runtimeContext) {
+    this.stepName = stepName;
     this.delegate = delegate;
-    container = new FlinkMetricContainer(stepName, runtimeContext);
+    container = new FlinkMetricContainer(runtimeContext);
   }
 
   @Override
   public void startBundle() {
     try (Closeable ignored =
-             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer())) {
+             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) {
       delegate.startBundle();
     } catch (IOException e) {
       throw new RuntimeException(e);
@@ -58,7 +60,7 @@ public void startBundle() {
   @Override
   public void processElement(final WindowedValue<InputT> elem) {
     try (Closeable ignored =
-             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer())) {
+             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) {
       delegate.processElement(elem);
     } catch (IOException e) {
       throw new RuntimeException(e);
@@ -69,7 +71,7 @@ public void processElement(final WindowedValue<InputT> elem) {
   public void onTimer(final String timerId, final BoundedWindow window, final Instant timestamp,
                       final TimeDomain timeDomain) {
     try (Closeable ignored =
-             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer())) {
+             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) {
       delegate.onTimer(timerId, window, timestamp, timeDomain);
     } catch (IOException e) {
       throw new RuntimeException(e);
@@ -79,7 +81,7 @@ public void onTimer(final String timerId, final BoundedWindow window, final Inst
   @Override
   public void finishBundle() {
     try (Closeable ignored =
-             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer())) {
+             MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) {
       delegate.finishBundle();
     } catch (IOException e) {
       throw new RuntimeException(e);
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java
index d020f69f2d80..f81205e9b8cc 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java
@@ -17,19 +17,24 @@
  */
 package org.apache.beam.runners.flink.metrics;
 
+import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.beam.sdk.metrics.DistributionData;
-import org.apache.beam.sdk.metrics.GaugeData;
-import org.apache.beam.sdk.metrics.MetricKey;
-import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricUpdates;
+import org.apache.beam.sdk.metrics.DistributionResult;
+import org.apache.beam.sdk.metrics.GaugeResult;
+import org.apache.beam.sdk.metrics.MetricQueryResults;
+import org.apache.beam.sdk.metrics.MetricResult;
+import org.apache.beam.sdk.metrics.MetricResults;
 import org.apache.beam.sdk.metrics.MetricsContainer;
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
+import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.LongCounter;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Gauge;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Helper class for holding a {@link MetricsContainer} and forwarding Beam metrics to
@@ -37,46 +42,61 @@
  */
 public class FlinkMetricContainer {
 
+  public static final String ACCUMULATOR_NAME = "__metricscontainers";
+
+  private static final Logger LOG = LoggerFactory.getLogger(FlinkMetricContainer.class);
+
   private static final String METRIC_KEY_SEPARATOR = "__";
-  static final String COUNTER_PREFIX = "__counter";
-  static final String DISTRIBUTION_PREFIX = "__distribution";
-  static final String GAUGE_PREFIX = "__gauge";
+  private static final String COUNTER_PREFIX = "__counter";
+  private static final String DISTRIBUTION_PREFIX = "__distribution";
+  private static final String GAUGE_PREFIX = "__gauge";
 
-  private final MetricsContainer metricsContainer;
   private final RuntimeContext runtimeContext;
   private final Map<String, Counter> flinkCounterCache;
   private final Map<String, FlinkDistributionGauge> flinkDistributionGaugeCache;
   private final Map<String, FlinkGauge> flinkGaugeCache;
+  private final MetricsAccumulator metricsAccumulator;
 
-  public FlinkMetricContainer(String stepName, RuntimeContext runtimeContext) {
-    metricsContainer = new MetricsContainer(stepName);
+  public FlinkMetricContainer(RuntimeContext runtimeContext) {
     this.runtimeContext = runtimeContext;
-    flinkCounterCache = new HashMap<>();
-    flinkDistributionGaugeCache = new HashMap<>();
-    flinkGaugeCache = new HashMap<>();
+    this.flinkCounterCache = new HashMap<>();
+    this.flinkDistributionGaugeCache = new HashMap<>();
+    this.flinkGaugeCache = new HashMap<>();
+
+    Accumulator<MetricsContainerStepMap, MetricsContainerStepMap> metricsAccumulator =
+        runtimeContext.getAccumulator(ACCUMULATOR_NAME);
+    if (metricsAccumulator == null) {
+      metricsAccumulator = new MetricsAccumulator();
+      try {
+        runtimeContext.addAccumulator(ACCUMULATOR_NAME, metricsAccumulator);
+      } catch (Exception e) {
+        LOG.error("Failed to create metrics accumulator.", e);
+      }
+    }
+    this.metricsAccumulator = (MetricsAccumulator) metricsAccumulator;
   }
 
-  public MetricsContainer getMetricsContainer() {
-    return metricsContainer;
+  MetricsContainer getMetricsContainer(String stepName) {
+    return metricsAccumulator != null
+        ? metricsAccumulator.getLocalValue().getContainer(stepName)
+        : null;
   }
 
-  public void updateMetrics() {
-    // update metrics
-    MetricUpdates updates = metricsContainer.getUpdates();
-    if (updates != null) {
-      updateCounters(updates.counterUpdates());
-      updateDistributions(updates.distributionUpdates());
-      updateGauge(updates.gaugeUpdates());
-      metricsContainer.commitUpdates();
-    }
+  void updateMetrics() {
+    MetricResults metricResults =
+        asAttemptedOnlyMetricResults(metricsAccumulator.getLocalValue());
+    MetricQueryResults metricQueryResults =
+        metricResults.queryMetrics(MetricsFilter.builder().build());
+    updateCounters(metricQueryResults.counters());
+    updateDistributions(metricQueryResults.distributions());
+    updateGauge(metricQueryResults.gauges());
   }
 
-  private void updateCounters(Iterable<MetricUpdates.MetricUpdate<Long>> updates) {
-
-    for (MetricUpdates.MetricUpdate<Long> metricUpdate : updates) {
+  private void updateCounters(Iterable<MetricResult<Long>> counters) {
+    for (MetricResult<Long> metricResult : counters) {
+      String flinkMetricName = getFlinkMetricNameString(COUNTER_PREFIX, metricResult);
 
-      String flinkMetricName = getFlinkMetricNameString(COUNTER_PREFIX, metricUpdate.getKey());
-      Long update = metricUpdate.getUpdate();
+      Long update = metricResult.attempted();
 
       // update flink metric
       Counter counter = flinkCounterCache.get(flinkMetricName);
@@ -86,26 +106,15 @@ private void updateCounters(Iterable<MetricUpdates.MetricUpdate<Long>> updates)
       }
       counter.dec(counter.getCount());
       counter.inc(update);
-
-      // update flink accumulator
-      Accumulator<Long, Long> accumulator = runtimeContext.getAccumulator(flinkMetricName);
-      if (accumulator == null) {
-        accumulator = new LongCounter(update);
-        runtimeContext.addAccumulator(flinkMetricName, accumulator);
-      } else {
-        accumulator.resetLocal();
-        accumulator.add(update);
-      }
     }
   }
 
-  private void updateDistributions(Iterable<MetricUpdates.MetricUpdate<DistributionData>> updates) {
-
-    for (MetricUpdates.MetricUpdate<DistributionData> metricUpdate : updates) {
-
+  private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) {
+    for (MetricResult<DistributionResult> metricResult : distributions) {
       String flinkMetricName =
-          getFlinkMetricNameString(DISTRIBUTION_PREFIX, metricUpdate.getKey());
-      DistributionData update = metricUpdate.getUpdate();
+          getFlinkMetricNameString(DISTRIBUTION_PREFIX, metricResult);
+
+      DistributionResult update = metricResult.attempted();
 
       // update flink metric
       FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricName);
@@ -116,26 +125,15 @@ private void updateDistributions(Iterable<MetricUpdates.MetricUpdate<Distributio
       } else {
         gauge.update(update);
       }
-
-      // update flink accumulator
-      Accumulator<DistributionData, DistributionData> accumulator =
-          runtimeContext.getAccumulator(flinkMetricName);
-      if (accumulator == null) {
-        accumulator = new FlinkDistributionDataAccumulator(update);
-        runtimeContext.addAccumulator(flinkMetricName, accumulator);
-      } else {
-        accumulator.resetLocal();
-        accumulator.add(update);
-      }
     }
   }
 
-  private void updateGauge(Iterable<MetricUpdates.MetricUpdate<GaugeData>> updates) {
-    for (MetricUpdates.MetricUpdate<GaugeData> metricUpdate : updates) {
-
+  private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) {
+    for (MetricResult<GaugeResult> metricResult : gauges) {
       String flinkMetricName =
-          getFlinkMetricNameString(GAUGE_PREFIX, metricUpdate.getKey());
-      GaugeData update = metricUpdate.getUpdate();
+          getFlinkMetricNameString(GAUGE_PREFIX, metricResult);
+
+      GaugeResult update = metricResult.attempted();
 
       // update flink metric
       FlinkGauge gauge = flinkGaugeCache.get(flinkMetricName);
@@ -146,170 +144,55 @@ private void updateGauge(Iterable<MetricUpdates.MetricUpdate<GaugeData>> updates
       } else {
         gauge.update(update);
       }
-
-      // update flink accumulator
-      Accumulator<GaugeData, GaugeData> accumulator =
-          runtimeContext.getAccumulator(flinkMetricName);
-      if (accumulator == null) {
-        accumulator = new FlinkGaugeAccumulator(update);
-        runtimeContext.addAccumulator(flinkMetricName, accumulator);
-      }
-      accumulator.resetLocal();
-      accumulator.add(update);
     }
   }
 
-  private static String getFlinkMetricNameString(String prefix, MetricKey key) {
+  private static String getFlinkMetricNameString(String prefix, MetricResult<?> metricResult) {
     return prefix
-        + METRIC_KEY_SEPARATOR + key.stepName()
-        + METRIC_KEY_SEPARATOR + key.metricName().namespace()
-        + METRIC_KEY_SEPARATOR + key.metricName().name();
-  }
-
-  static MetricKey parseMetricKey(String flinkMetricName) {
-    String[] arr = flinkMetricName.split(METRIC_KEY_SEPARATOR);
-    return MetricKey.create(arr[2], MetricName.named(arr[3], arr[4]));
+        + METRIC_KEY_SEPARATOR + metricResult.step()
+        + METRIC_KEY_SEPARATOR + metricResult.name().namespace()
+        + METRIC_KEY_SEPARATOR + metricResult.name().name();
   }
 
   /**
-   * Flink {@link Gauge} for {@link DistributionData}.
+   * Flink {@link Gauge} for {@link DistributionResult}.
    */
-  public static class FlinkDistributionGauge implements Gauge<DistributionData> {
+  public static class FlinkDistributionGauge implements Gauge<DistributionResult> {
 
-    DistributionData data;
+    DistributionResult data;
 
-    FlinkDistributionGauge(DistributionData data) {
+    FlinkDistributionGauge(DistributionResult data) {
       this.data = data;
     }
 
-    void update(DistributionData data) {
+    void update(DistributionResult data) {
       this.data = data;
     }
 
     @Override
-    public DistributionData getValue() {
+    public DistributionResult getValue() {
       return data;
     }
   }
 
   /**
-   * Flink {@link Gauge} for {@link GaugeData}.
+   * Flink {@link Gauge} for {@link GaugeResult}.
    */
-  public static class FlinkGauge implements Gauge<GaugeData> {
+  public static class FlinkGauge implements Gauge<GaugeResult> {
 
-    GaugeData data;
+    GaugeResult data;
 
-    FlinkGauge(GaugeData data) {
+    FlinkGauge(GaugeResult data) {
       this.data = data;
     }
 
-    void update(GaugeData update) {
-      this.data = data.combine(update);
+    void update(GaugeResult update) {
+      this.data = update;
     }
 
     @Override
-    public GaugeData getValue() {
+    public GaugeResult getValue() {
       return data;
     }
   }
-
-  /**
-   * Flink {@link Accumulator} for {@link GaugeData}.
-   */
-  public static class FlinkDistributionDataAccumulator implements
-      Accumulator<DistributionData, DistributionData> {
-
-    private static final long serialVersionUID = 1L;
-
-    private DistributionData data;
-
-    public FlinkDistributionDataAccumulator(DistributionData data) {
-      this.data = data;
-    }
-
-    @Override
-    public void add(DistributionData value) {
-      if (data == null) {
-        this.data = value;
-      } else {
-        this.data = this.data.combine(value);
-      }
-    }
-
-    @Override
-    public DistributionData getLocalValue() {
-      return data;
-    }
-
-    @Override
-    public void resetLocal() {
-      data = null;
-    }
-
-    @Override
-    public void merge(Accumulator<DistributionData, DistributionData> other) {
-      data = data.combine(other.getLocalValue());
-    }
-
-    @Override
-    public Accumulator<DistributionData, DistributionData> clone() {
-      try {
-        super.clone();
-      } catch (CloneNotSupportedException e) {
-        throw new RuntimeException(e);
-      }
-
-      return new FlinkDistributionDataAccumulator(
-          DistributionData.create(data.sum(), data.count(), data.min(), data.max()));
-    }
-  }
-
-  /**
-   * Flink {@link Accumulator} for {@link GaugeData}.
-   */
-  public static class FlinkGaugeAccumulator implements Accumulator<GaugeData, GaugeData> {
-
-    private GaugeData data;
-
-    public FlinkGaugeAccumulator(GaugeData data) {
-      this.data = data;
-    }
-
-    @Override
-    public void add(GaugeData value) {
-      if (data == null) {
-        this.data = value;
-      } else {
-        this.data = this.data.combine(value);
-      }
-    }
-
-    @Override
-    public GaugeData getLocalValue() {
-      return data;
-    }
-
-    @Override
-    public void resetLocal() {
-      this.data = null;
-    }
-
-    @Override
-    public void merge(Accumulator<GaugeData, GaugeData> other) {
-      data = data.combine(other.getLocalValue());
-    }
-
-    @Override
-    public Accumulator<GaugeData, GaugeData> clone() {
-      try {
-        super.clone();
-      } catch (CloneNotSupportedException e) {
-        throw new RuntimeException(e);
-      }
-
-      return new FlinkGaugeAccumulator(
-          GaugeData.create(data.value()));
-    }
-  }
-
 }
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricResults.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricResults.java
deleted file mode 100644
index 9e1430bc03e4..000000000000
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricResults.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.flink.metrics;
-
-
-import static org.apache.beam.runners.flink.metrics.FlinkMetricContainer.COUNTER_PREFIX;
-import static org.apache.beam.runners.flink.metrics.FlinkMetricContainer.DISTRIBUTION_PREFIX;
-import static org.apache.beam.runners.flink.metrics.FlinkMetricContainer.GAUGE_PREFIX;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.metrics.DistributionData;
-import org.apache.beam.sdk.metrics.DistributionResult;
-import org.apache.beam.sdk.metrics.GaugeData;
-import org.apache.beam.sdk.metrics.GaugeResult;
-import org.apache.beam.sdk.metrics.MetricFiltering;
-import org.apache.beam.sdk.metrics.MetricKey;
-import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricQueryResults;
-import org.apache.beam.sdk.metrics.MetricResult;
-import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.metrics.MetricsFilter;
-
-/**
- * Implementation of {@link MetricResults} for the Flink Runner.
- */
-public class FlinkMetricResults extends MetricResults {
-
-  private Map<String, Object> accumulators;
-
-  public FlinkMetricResults(Map<String, Object> accumulators) {
-    this.accumulators = accumulators;
-  }
-
-  @Override
-  public MetricQueryResults queryMetrics(MetricsFilter filter) {
-    return new FlinkMetricQueryResults(filter);
-  }
-
-  private class FlinkMetricQueryResults implements MetricQueryResults {
-
-    private MetricsFilter filter;
-
-    FlinkMetricQueryResults(MetricsFilter filter) {
-      this.filter = filter;
-    }
-
-    @Override
-    public Iterable<MetricResult<Long>> counters() {
-      List<MetricResult<Long>> result = new ArrayList<>();
-      for (Map.Entry<String, Object> accumulator : accumulators.entrySet()) {
-        if (accumulator.getKey().startsWith(COUNTER_PREFIX)) {
-          MetricKey metricKey = FlinkMetricContainer.parseMetricKey(accumulator.getKey());
-          if (MetricFiltering.matches(filter, metricKey)) {
-            result.add(new FlinkMetricResult<>(
-                metricKey.metricName(), metricKey.stepName(), (Long) accumulator.getValue()));
-          }
-        }
-      }
-      return result;
-    }
-
-    @Override
-    public Iterable<MetricResult<DistributionResult>> distributions() {
-      List<MetricResult<DistributionResult>> result = new ArrayList<>();
-      for (Map.Entry<String, Object> accumulator : accumulators.entrySet()) {
-        if (accumulator.getKey().startsWith(DISTRIBUTION_PREFIX)) {
-          MetricKey metricKey = FlinkMetricContainer.parseMetricKey(accumulator.getKey());
-          DistributionData data = (DistributionData) accumulator.getValue();
-          if (MetricFiltering.matches(filter, metricKey)) {
-            result.add(new FlinkMetricResult<>(
-                metricKey.metricName(), metricKey.stepName(), data.extractResult()));
-          }
-        }
-      }
-      return result;
-    }
-
-    @Override
-    public Iterable<MetricResult<GaugeResult>> gauges() {
-      List<MetricResult<GaugeResult>> result = new ArrayList<>();
-      for (Map.Entry<String, Object> accumulator : accumulators.entrySet()) {
-        if (accumulator.getKey().startsWith(GAUGE_PREFIX)) {
-          MetricKey metricKey = FlinkMetricContainer.parseMetricKey(accumulator.getKey());
-          GaugeData data = (GaugeData) accumulator.getValue();
-          if (MetricFiltering.matches(filter, metricKey)) {
-            result.add(new FlinkMetricResult<>(
-                metricKey.metricName(), metricKey.stepName(), data.extractResult()));
-          }
-        }
-      }
-      return result;
-    }
-
-  }
-
-  private static class FlinkMetricResult<T> implements MetricResult<T> {
-    private final MetricName name;
-    private final String step;
-    private final T result;
-
-    FlinkMetricResult(MetricName name, String step, T result) {
-      this.name = name;
-      this.step = step;
-      this.result = result;
-    }
-
-    @Override
-    public MetricName name() {
-      return name;
-    }
-
-    @Override
-    public String step() {
-      return step;
-    }
-
-    @Override
-    public T committed() {
-      throw new UnsupportedOperationException("Flink runner does not currently support committed"
-          + " metrics results. Please use 'attempted' instead.");
-    }
-
-    @Override
-    public T attempted() {
-      return result;
-    }
-  }
-
-}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java
new file mode 100644
index 000000000000..a9dc2ce92be4
--- /dev/null
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink.metrics;
+
+import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
+import org.apache.flink.api.common.accumulators.Accumulator;
+import org.apache.flink.api.common.accumulators.SimpleAccumulator;
+
+/**
+ * Accumulator of {@link MetricsContainerStepMap}.
+ */
+public class MetricsAccumulator implements SimpleAccumulator<MetricsContainerStepMap> {
+  private MetricsContainerStepMap metricsContainers = new MetricsContainerStepMap();
+
+  @Override
+  public void add(MetricsContainerStepMap value) {
+    metricsContainers.updateAll(value);
+  }
+
+  @Override
+  public MetricsContainerStepMap getLocalValue() {
+    return metricsContainers;
+  }
+
+  @Override
+  public void resetLocal() {
+    this.metricsContainers = new MetricsContainerStepMap();
+  }
+
+  @Override
+  public void merge(Accumulator<MetricsContainerStepMap, MetricsContainerStepMap> other) {
+    this.add(other.getLocalValue());
+  }
+
+  @Override
+  public Accumulator<MetricsContainerStepMap, MetricsContainerStepMap> clone() {
+    try {
+      super.clone();
+    } catch (CloneNotSupportedException ignored) {
+    }
+    MetricsAccumulator metricsAccumulator = new MetricsAccumulator();
+    metricsAccumulator.getLocalValue().updateAll(this.getLocalValue());
+    return metricsAccumulator;
+  }
+}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java
index 38263d98f687..64738cce2af8 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java
@@ -32,13 +32,16 @@
  */
 public class ReaderInvocationUtil<OutputT, ReaderT extends Source.Reader<OutputT>> {
 
+  private final String stepName;
   private final FlinkMetricContainer container;
   private final Boolean enableMetrics;
 
   public ReaderInvocationUtil(
+      String stepName,
       PipelineOptions options,
       FlinkMetricContainer container) {
     FlinkPipelineOptions flinkPipelineOptions = options.as(FlinkPipelineOptions.class);
+    this.stepName = stepName;
     enableMetrics = flinkPipelineOptions.getEnableMetrics();
     this.container = container;
   }
@@ -46,7 +49,7 @@ public ReaderInvocationUtil(
   public boolean invokeStart(ReaderT reader) throws IOException {
     if (enableMetrics) {
       try (Closeable ignored =
-               MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer())) {
+               MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) {
         boolean result = reader.start();
         container.updateMetrics();
         return result;
@@ -59,7 +62,7 @@ public boolean invokeStart(ReaderT reader) throws IOException {
   public boolean invokeAdvance(ReaderT reader) throws IOException {
     if (enableMetrics) {
       try (Closeable ignored =
-               MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer())) {
+               MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) {
         boolean result = reader.advance();
         container.updateMetrics();
         return result;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java
index f2b81fc173b6..27e691231fae 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java
@@ -71,9 +71,13 @@ public void configure(Configuration configuration) {
 
   @Override
   public void open(SourceInputSplit<T> sourceInputSplit) throws IOException {
-    FlinkMetricContainer metricContainer = new FlinkMetricContainer(stepName, getRuntimeContext());
+    FlinkMetricContainer metricContainer = new FlinkMetricContainer(getRuntimeContext());
+
     readerInvoker =
-        new ReaderInvocationUtil<>(serializedOptions.getPipelineOptions(), metricContainer);
+        new ReaderInvocationUtil<>(
+            stepName,
+            serializedOptions.getPipelineOptions(),
+            metricContainer);
 
     reader = ((BoundedSource<T>) sourceInputSplit.getSource()).createReader(options);
     inputAvailable = readerInvoker.invokeStart(reader);
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java
index a14268592a5b..6d756880f618 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/BoundedSourceWrapper.java
@@ -104,9 +104,13 @@ public void run(SourceContext<WindowedValue<OutputT>> ctx) throws Exception {
         numSubtasks,
         localSources);
 
-    FlinkMetricContainer metricContainer = new FlinkMetricContainer(stepName, getRuntimeContext());
+    FlinkMetricContainer metricContainer = new FlinkMetricContainer(getRuntimeContext());
+
     ReaderInvocationUtil<OutputT, BoundedSource.BoundedReader<OutputT>> readerInvoker =
-        new ReaderInvocationUtil<>(serializedOptions.getPipelineOptions(), metricContainer);
+        new ReaderInvocationUtil<>(
+            stepName,
+            serializedOptions.getPipelineOptions(),
+            metricContainer);
 
     readers = new ArrayList<>();
     // initialize readers from scratch
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
index ee20fd591ace..b9c431d89377 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
@@ -214,10 +214,13 @@ public void run(SourceContext<WindowedValue<OutputT>> ctx) throws Exception {
 
     context = ctx;
 
-    FlinkMetricContainer metricContainer = new FlinkMetricContainer(stepName, getRuntimeContext());
-    ReaderInvocationUtil<OutputT, UnboundedSource.UnboundedReader<OutputT>> readerInvoker =
-        new ReaderInvocationUtil<>(serializedOptions.getPipelineOptions(), metricContainer);
+    FlinkMetricContainer metricContainer = new FlinkMetricContainer(getRuntimeContext());
 
+    ReaderInvocationUtil<OutputT, UnboundedSource.UnboundedReader<OutputT>> readerInvoker =
+        new ReaderInvocationUtil<>(
+            stepName,
+            serializedOptions.getPipelineOptions(),
+            metricContainer);
 
     if (localReaders.size() == 0) {
       // do nothing, but still look busy ...

From eb043d06df0da4a47d16047f1f050e41114ac424 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Sat, 6 May 2017 05:13:34 -0700
Subject: [PATCH 102/387] Register the TestDataflowRunner via registrar

---
 .../jenkins/job_beam_PostCommit_Java_MavenInstall.groovy      | 2 +-
 .../job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy  | 2 +-
 .test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy   | 2 +-
 examples/java/pom.xml                                         | 4 ++--
 .../beam/runners/dataflow/DataflowPipelineRegistrar.java      | 3 ++-
 .../beam/runners/dataflow/DataflowPipelineRegistrarTest.java  | 2 +-
 .../main/java/org/apache/beam/sdk/testing/TestPipeline.java   | 2 +-
 7 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
index 51aedc31fc98..2f05c38e06bc 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
@@ -44,5 +44,5 @@ mavenJob('beam_PostCommit_Java_MavenInstall') {
           'Run Java PostCommit')
 
   // Maven goals for this job.
-  goals('-B -e -P release,dataflow-runner clean install coveralls:report -DrepoToken=$COVERALLS_REPO_TOKEN -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'')
+  goals('-B -e -P release,dataflow-runner clean install coveralls:report -DrepoToken=$COVERALLS_REPO_TOKEN -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=TestDataflowRunner" ]\'')
 }
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
index 33235ff833c2..2c739e359e3d 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
@@ -41,5 +41,5 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') {
     'Run Dataflow ValidatesRunner')
 
   // Maven goals for this job.
-  goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner", "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'')
+  goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestDataflowRunner", "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'')
 }
diff --git a/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
index f2c3ff0740d5..7284acd8ad14 100644
--- a/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
@@ -41,5 +41,5 @@ mavenJob('beam_Release_NightlySnapshot') {
       'dev@beam.apache.org')
 
   // Maven goals for this job.
-  goals('-B -e clean deploy -P release,dataflow-runner -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'')
+  goals('-B -e clean deploy -P release,dataflow-runner -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=TestDataflowRunner" ]\'')
 }
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index a6ef5d1ee1c7..21096f74bf16 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -263,7 +263,7 @@
                       [
                       "--project=apache-beam-testing",
                       "--tempRoot=gs://temp-storage-for-end-to-end-tests",
-                      "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner"
+                      "--runner=TestDataflowRunner"
                       ]
                     </beamTestPipelineOptions>
                   </systemPropertyVariables>
@@ -287,7 +287,7 @@
                       [
                       "--project=apache-beam-testing",
                       "--tempRoot=gs://temp-storage-for-end-to-end-tests",
-                      "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner",
+                      "--runner=TestDataflowRunner",
                       "--streaming=true"
                       ]
                     </beamTestPipelineOptions>
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
index b6802bb4b0aa..f36930fa98a6 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
@@ -20,6 +20,7 @@
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.testing.TestDataflowRunner;
 import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
@@ -52,7 +53,7 @@ public static class Runner implements PipelineRunnerRegistrar {
     @Override
     public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
       return ImmutableList.<Class<? extends PipelineRunner<?>>>of(
-          DataflowRunner.class);
+          DataflowRunner.class, TestDataflowRunner.class);
     }
   }
 }
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java
index 31f9281f9551..728fc71cc782 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java
@@ -41,7 +41,7 @@ public void testCorrectOptionsAreReturned() {
 
   @Test
   public void testCorrectRunnersAreReturned() {
-    assertEquals(ImmutableList.of(DataflowRunner.class),
+    assertEquals(ImmutableList.of(DataflowRunner.class, TestDataflowRunner.class),
         new DataflowPipelineRegistrar.Runner().getPipelineRunners());
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index 96cae51765e6..e04c2f835ab5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -72,7 +72,7 @@
  * <li>System property "beamTestPipelineOptions" must contain a JSON delimited list of pipeline
  *     options. For example:
  *     <pre>{@code [
- *     "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner",
+ *     "--runner=TestDataflowRunner",
  *     "--project=mygcpproject",
  *     "--stagingLocation=gs://mygcsbucket/path"
  *     ]}</pre>

From b46e7b9bb113a5b1b75c7aca9de8cbabdd75ff6f Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Sat, 6 May 2017 05:22:03 -0700
Subject: [PATCH 103/387] Move TestDataflowRunner into dataflow package

---
 .../dataflow/DataflowPipelineRegistrar.java   |  1 -
 .../TestDataflowPipelineOptions.java          |  2 +-
 .../{testing => }/TestDataflowRunner.java     |  5 +---
 .../dataflow/testing/package-info.java        | 24 -------------------
 .../runners/dataflow/DataflowMetricsTest.java |  1 -
 .../dataflow/DataflowPipelineJobTest.java     |  1 -
 .../{testing => }/TestDataflowRunnerTest.java |  5 +---
 7 files changed, 3 insertions(+), 36 deletions(-)
 rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/{testing => }/TestDataflowPipelineOptions.java (95%)
 rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/{testing => }/TestDataflowRunner.java (98%)
 delete mode 100644 runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/package-info.java
 rename runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/{testing => }/TestDataflowRunnerTest.java (99%)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
index f36930fa98a6..15855f93e273 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java
@@ -20,7 +20,6 @@
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
-import org.apache.beam.runners.dataflow.testing.TestDataflowRunner;
 import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowPipelineOptions.java
similarity index 95%
rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineOptions.java
rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowPipelineOptions.java
index 12f7b39035b9..a8acc764c3ae 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowPipelineOptions.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.runners.dataflow.testing;
+package org.apache.beam.runners.dataflow;
 
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
 import org.apache.beam.sdk.testing.TestPipeline;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
similarity index 98%
rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
index ce9191586d1f..b81b48771684 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.runners.dataflow.testing;
+package org.apache.beam.runners.dataflow;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 
@@ -30,9 +30,6 @@
 import java.math.BigDecimal;
 import java.util.List;
 import javax.annotation.Nullable;
-import org.apache.beam.runners.dataflow.DataflowClient;
-import org.apache.beam.runners.dataflow.DataflowPipelineJob;
-import org.apache.beam.runners.dataflow.DataflowRunner;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
 import org.apache.beam.runners.dataflow.util.MonitoringUtil;
 import org.apache.beam.runners.dataflow.util.MonitoringUtil.JobMessagesHandler;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/package-info.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/package-info.java
deleted file mode 100644
index 9683df0b08e7..000000000000
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Provides utilities for integration testing and {@link
- * org.apache.beam.sdk.testing.ValidatesRunner} tests of the Google Cloud Dataflow
- * runner.
- */
-package org.apache.beam.runners.dataflow.testing;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
index aabdd84c124b..7e883005aff7 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
@@ -37,7 +37,6 @@
 import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.math.BigDecimal;
-import org.apache.beam.runners.dataflow.testing.TestDataflowPipelineOptions;
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
index f868a174ba65..df894d2cb33a 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
@@ -48,7 +48,6 @@
 import java.util.NavigableMap;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
-import org.apache.beam.runners.dataflow.testing.TestDataflowPipelineOptions;
 import org.apache.beam.runners.dataflow.util.MonitoringUtil;
 import org.apache.beam.runners.dataflow.util.TimeUtil;
 import org.apache.beam.sdk.Pipeline;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
similarity index 99%
rename from runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
rename to runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
index eb068e67c3b3..bf157474b677 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.runners.dataflow.testing;
+package org.apache.beam.runners.dataflow;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -44,9 +44,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import org.apache.beam.runners.dataflow.DataflowClient;
-import org.apache.beam.runners.dataflow.DataflowPipelineJob;
-import org.apache.beam.runners.dataflow.DataflowRunner;
 import org.apache.beam.runners.dataflow.util.MonitoringUtil;
 import org.apache.beam.runners.dataflow.util.MonitoringUtil.JobMessagesHandler;
 import org.apache.beam.runners.dataflow.util.TimeUtil;

From 78a99be1c46656996081744e52dac3d619cb4fe7 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Fri, 5 May 2017 15:18:52 -0700
Subject: [PATCH 104/387] Remove context from size estimate operations.

---
 .../org/apache/beam/sdk/coders/Coder.java     | 52 ++++---------------
 1 file changed, 10 insertions(+), 42 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index eeafbd228603..d140e896e0a0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -301,39 +301,9 @@ public Object structuralValue(T value) {
    *         unless it is overridden. This is considered expensive.
    */
   public boolean isRegisterByteSizeObserverCheap(T value) {
-    return isRegisterByteSizeObserverCheap(value, Context.NESTED);
-  }
-
-  /**
-   * {@inheritDoc}
-   *
-   * <p>Not intended to be called by user code, but instead by
-   * {@link PipelineRunner}
-   * implementations.
-   *
-   * @return {@code false} unless it is overridden. {@link StructuredCoder#registerByteSizeObserver}
-   *         invokes {@link #getEncodedElementByteSize} which requires re-encoding an element
-   *         unless it is overridden. This is considered expensive.
-   */
-  @Deprecated
-  public boolean isRegisterByteSizeObserverCheap(T value, Context context) {
     return false;
   }
 
-  /**
-   * Returns the size in bytes of the encoded value using this coder.
-   */
-  protected long getEncodedElementByteSize(T value, Context context)
-      throws Exception {
-    try (CountingOutputStream os = new CountingOutputStream(ByteStreams.nullOutputStream())) {
-      encode(value, os, context);
-      return os.getCount();
-    } catch (Exception exn) {
-      throw new IllegalArgumentException(
-          "Unable to encode element '" + value + "' with coder '" + this + "'.", exn);
-    }
-  }
-
   /**
    * Notifies the {@code ElementByteSizeObserver} about the byte size
    * of the encoded value using this {@code Coder}.
@@ -347,22 +317,20 @@ protected long getEncodedElementByteSize(T value, Context context)
    */
   public void registerByteSizeObserver(T value, ElementByteSizeObserver observer)
       throws Exception {
-    registerByteSizeObserver(value, observer, Context.NESTED);
+    observer.update(getEncodedElementByteSize(value));
   }
 
   /**
-   * Notifies the {@code ElementByteSizeObserver} about the byte size
-   * of the encoded value using this {@code Coder}.
-   *
-   * <p>Not intended to be called by user code, but instead by
-   * {@link PipelineRunner}
-   * implementations.
+   * Returns the size in bytes of the encoded value using this coder.
    */
-  @Deprecated
-  public void registerByteSizeObserver(
-      T value, ElementByteSizeObserver observer, Context context)
-      throws Exception {
-    observer.update(getEncodedElementByteSize(value, context));
+  protected long getEncodedElementByteSize(T value) throws Exception {
+    try (CountingOutputStream os = new CountingOutputStream(ByteStreams.nullOutputStream())) {
+      encode(value, os);
+      return os.getCount();
+    } catch (Exception exn) {
+      throw new IllegalArgumentException(
+          "Unable to encode element '" + value + "' with coder '" + this + "'.", exn);
+    }
   }
 
   /**

From 55a7cd4f97f2e3fee709f034a0748284c9030df7 Mon Sep 17 00:00:00 2001
From: chinmaykolhatkar <chinmay@apache.org>
Date: Wed, 1 Mar 2017 16:59:46 +0530
Subject: [PATCH 105/387] [BEAM-831] ParDo Fusion of Apex Runner

---
 .../runners/apex/ApexPipelineOptions.java     |   5 +
 .../apex/translation/TranslationContext.java  | 103 ++++++++++++++++--
 2 files changed, 97 insertions(+), 11 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
index f37e87456fbb..92f6e8f03eeb 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexPipelineOptions.java
@@ -62,4 +62,9 @@ public interface ApexPipelineOptions extends PipelineOptions, java.io.Serializab
   @Default.String("classpath:/beam-runners-apex.properties")
   String getConfigFile();
 
+  @Description("configure whether to perform ParDo fusion")
+  void setParDoFusionEnabled(boolean enabled);
+
+  @Default.Boolean(true)
+  boolean isParDoFusionEnabled();
 }
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index 9c20449d07e0..1224e254e801 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -37,6 +37,8 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -54,7 +56,8 @@ class TranslationContext {
 
   private final ApexPipelineOptions pipelineOptions;
   private AppliedPTransform<?, ?, ?> currentTransform;
-  private final Map<PCollection, Pair<OutputPort<?>, List<InputPort<?>>>> streams = new HashMap<>();
+  private final Map<PCollection, Pair<OutputPortInfo, List<InputPortInfo>>> streams =
+          new HashMap<>();
   private final Map<String, Operator> operators = new HashMap<>();
   private final Map<PCollectionView<?>, PInput> viewInputs = new HashMap<>();
   private Map<PInput, PInput> aliasCollections = new HashMap<>();
@@ -122,8 +125,10 @@ public void addOperator(Operator operator, Map<PCollection<?>, OutputPort<?>> po
         addOperator(operator, portEntry.getValue(), portEntry.getKey());
         first = false;
       } else {
-        this.streams.put(portEntry.getKey(), (Pair) new ImmutablePair<>(portEntry.getValue(),
-            new ArrayList<>()));
+        this.streams.put(portEntry.getKey(),
+                         (Pair) new ImmutablePair<>(new OutputPortInfo(portEntry.getValue(),
+                                                                       getCurrentTransform()),
+                                                    new ArrayList<>()));
       }
     }
   }
@@ -142,16 +147,19 @@ public void addOperator(Operator operator, OutputPort port, PCollection output)
       name = getCurrentTransform().getFullName() + i;
     }
     this.operators.put(name, operator);
-    this.streams.put(output, (Pair) new ImmutablePair<>(port, new ArrayList<>()));
+    this.streams.put(output, (Pair) new ImmutablePair<>(
+            new OutputPortInfo(port, getCurrentTransform()),
+            new ArrayList<>()));
   }
 
   public void addStream(PInput input, InputPort inputPort) {
     while (aliasCollections.containsKey(input)) {
       input = aliasCollections.get(input);
     }
-    Pair<OutputPort<?>, List<InputPort<?>>> stream = this.streams.get(input);
-    checkArgument(stream != null, "no upstream operator defined for %s", input);
-    stream.getRight().add(inputPort);
+
+    Pair<OutputPortInfo, List<InputPortInfo>> stream = this.streams.get(input);
+    checkArgument(stream != null, "no upstream operator defined for " + input);
+    stream.getRight().add(new InputPortInfo(inputPort, getCurrentTransform()));
   }
 
   /**
@@ -168,13 +176,23 @@ public void populateDAG(DAG dag) {
     for (Map.Entry<String, Operator> nameAndOperator : this.operators.entrySet()) {
       dag.addOperator(nameAndOperator.getKey(), nameAndOperator.getValue());
     }
+
     int streamIndex = 0;
-    for (Map.Entry<PCollection, Pair<OutputPort<?>, List<InputPort<?>>>> streamEntry : this.
+    for (Map.Entry<PCollection, Pair<OutputPortInfo, List<InputPortInfo>>> streamEntry : this.
         streams.entrySet()) {
-      List<InputPort<?>> sinksList = streamEntry.getValue().getRight();
-      InputPort[] sinks = sinksList.toArray(new InputPort[sinksList.size()]);
+      List<InputPortInfo> destInfo = streamEntry.getValue().getRight();
+      InputPort[] sinks = new InputPort[destInfo.size()];
+      for (int i = 0; i < sinks.length; i++) {
+        sinks[i] = destInfo.get(i).port;
+      }
+
       if (sinks.length > 0) {
-        dag.addStream("stream" + streamIndex++, streamEntry.getValue().getLeft(), sinks);
+        DAG.StreamMeta streamMeta = dag.addStream("stream" + streamIndex++,
+                                                  streamEntry.getValue().getLeft().port, sinks);
+        if (pipelineOptions.isParDoFusionEnabled()) {
+          optimizeStreams(streamMeta, streamEntry);
+        }
+
         for (InputPort port : sinks) {
           PCollection pc = streamEntry.getKey();
           Coder coder = pc.getCoder();
@@ -191,6 +209,49 @@ public void populateDAG(DAG dag) {
     }
   }
 
+  private void optimizeStreams(DAG.StreamMeta streamMeta,
+                               Map.Entry<PCollection,
+                                         Pair<OutputPortInfo, List<InputPortInfo>>> streamEntry) {
+    DAG.Locality loc = null;
+
+    List<InputPortInfo> sinks = streamEntry.getValue().getRight();
+    OutputPortInfo source = streamEntry.getValue().getLeft();
+    PTransform sourceTransform = source.transform.getTransform();
+    if (sourceTransform instanceof ParDo.Bound
+        || sourceTransform instanceof ParDo.BoundMulti) {
+      // Source is ParDo.. Check sink(s)
+      for (InputPortInfo sink : sinks) {
+        PTransform transform = sink.transform.getTransform();
+        if (transform instanceof ParDo.Bound) {
+          ParDo.Bound t = (ParDo.Bound) transform;
+          if (t.getSideInputs().size() > 0) {
+            loc = DAG.Locality.CONTAINER_LOCAL;
+            break;
+          } else {
+            loc = DAG.Locality.THREAD_LOCAL;
+          }
+        } else if (transform instanceof ParDo.BoundMulti) {
+          ParDo.BoundMulti t = (ParDo.BoundMulti) transform;
+          if (t.getSideInputs().size() > 0) {
+            loc = DAG.Locality.CONTAINER_LOCAL;
+            break;
+          } else {
+            loc = DAG.Locality.THREAD_LOCAL;
+          }
+        } else {
+          // Sink is not ParDo.. set null locality.
+          loc = null;
+          break;
+        }
+      }
+    } else {
+      // Source is not ParDo... set null locality
+      loc = null;
+    }
+
+    streamMeta.setLocality(loc);
+  }
+
   /**
    * Return the state backend for the pipeline translation.
    * @return
@@ -198,4 +259,24 @@ public void populateDAG(DAG dag) {
   public ApexStateBackend getStateBackend() {
     return new ApexStateInternals.ApexStateBackend();
   }
+
+  static class InputPortInfo {
+    InputPort port;
+    AppliedPTransform transform;
+
+    public InputPortInfo(InputPort port, AppliedPTransform transform) {
+      this.port = port;
+      this.transform = transform;
+    }
+  }
+
+  static class OutputPortInfo {
+    OutputPort port;
+    AppliedPTransform transform;
+
+    public OutputPortInfo(OutputPort port, AppliedPTransform transform) {
+      this.port = port;
+      this.transform = transform;
+    }
+  }
 }

From b64c8ce7dc55426e0c4110a64ce5ec8e83151ab0 Mon Sep 17 00:00:00 2001
From: Thomas Weise <thw@apache.org>
Date: Fri, 5 May 2017 06:45:34 -0700
Subject: [PATCH 106/387] [BEAM-831] Fix chaining, add test. closes #2216

---
 .../apache/beam/runners/apex/ApexRunner.java  | 11 +++--
 .../beam/runners/apex/TestApexRunner.java     |  8 ++++
 .../apex/translation/TranslationContext.java  | 26 ++++------
 .../beam/runners/apex/ApexRunnerTest.java     | 47 +++++++++++++++----
 .../FlattenPCollectionTranslatorTest.java     | 13 ++---
 5 files changed, 67 insertions(+), 38 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index a50e10ef53d8..366308ec6642 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -70,11 +70,11 @@
  * pipeline to an Apex DAG and executes it on an Apex cluster.
  *
  */
-@SuppressWarnings({"rawtypes", "unchecked"})
 public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
 
   private final ApexPipelineOptions options;
   public static final String CLASSPATH_SCHEME = "classpath";
+  protected boolean translateOnly = false;
 
   /**
    * TODO: this isn't thread safe and may cause issues when tests run in parallel
@@ -93,6 +93,7 @@ public static ApexRunner fromOptions(PipelineOptions options) {
     return new ApexRunner(apexPipelineOptions);
   }
 
+  @SuppressWarnings({"rawtypes"})
   private List<PTransformOverride> getOverrides() {
     return ImmutableList.<PTransformOverride>builder()
         .add(
@@ -156,7 +157,7 @@ public void populateDAG(DAG dag, Configuration conf) {
     }
 
     if (options.isEmbeddedExecution()) {
-      Launcher<AppHandle> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
+      EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
       Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
       launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
       if (options.isEmbeddedExecutionDebugMode()) {
@@ -166,11 +167,15 @@ public void populateDAG(DAG dag, Configuration conf) {
       Configuration conf = new Configuration(false);
       ApexYarnLauncher.addProperties(conf, configProperties);
       try {
+        if (translateOnly) {
+          launcher.prepareDAG(apexApp, conf);
+          return new ApexRunnerResult(launcher.getDAG(), null);
+        }
         ApexRunner.ASSERTION_ERROR.set(null);
         AppHandle apexAppResult = launcher.launchApp(apexApp, conf, launchAttributes);
         return new ApexRunnerResult(apexDAG.get(), apexAppResult);
       } catch (Exception e) {
-        Throwables.propagateIfPossible(e);
+        Throwables.throwIfUnchecked(e);
         throw new RuntimeException(e);
       }
     } else {
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
index e068db086442..b68d3da5891b 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/TestApexRunner.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.runners.apex;
 
+import com.datatorrent.api.DAG;
 import java.io.IOException;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineRunner;
@@ -44,6 +45,13 @@ public static TestApexRunner fromOptions(PipelineOptions options) {
     return new TestApexRunner(apexOptions);
   }
 
+  public static DAG translate(Pipeline pipeline, ApexPipelineOptions options) {
+    ApexRunner delegate = new ApexRunner(options);
+    delegate.translateOnly = true;
+    DAG dag = delegate.run(pipeline).getApexDAG();
+    return dag;
+  }
+
   @Override
   public ApexRunnerResult run(Pipeline pipeline) {
     ApexRunnerResult result = delegate.run(pipeline);
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index 1224e254e801..a5e30281a9e3 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -39,6 +39,7 @@
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -217,36 +218,27 @@ private void optimizeStreams(DAG.StreamMeta streamMeta,
     List<InputPortInfo> sinks = streamEntry.getValue().getRight();
     OutputPortInfo source = streamEntry.getValue().getLeft();
     PTransform sourceTransform = source.transform.getTransform();
-    if (sourceTransform instanceof ParDo.Bound
-        || sourceTransform instanceof ParDo.BoundMulti) {
-      // Source is ParDo.. Check sink(s)
+    if (sourceTransform instanceof ParDo.MultiOutput
+        || sourceTransform instanceof Window.Assign) {
+      // source qualifies for chaining, check sink(s)
       for (InputPortInfo sink : sinks) {
         PTransform transform = sink.transform.getTransform();
-        if (transform instanceof ParDo.Bound) {
-          ParDo.Bound t = (ParDo.Bound) transform;
-          if (t.getSideInputs().size() > 0) {
-            loc = DAG.Locality.CONTAINER_LOCAL;
-            break;
-          } else {
-            loc = DAG.Locality.THREAD_LOCAL;
-          }
-        } else if (transform instanceof ParDo.BoundMulti) {
-          ParDo.BoundMulti t = (ParDo.BoundMulti) transform;
+        if (transform instanceof ParDo.MultiOutput) {
+          ParDo.MultiOutput t = (ParDo.MultiOutput) transform;
           if (t.getSideInputs().size() > 0) {
             loc = DAG.Locality.CONTAINER_LOCAL;
             break;
           } else {
             loc = DAG.Locality.THREAD_LOCAL;
           }
+        } else if (transform instanceof Window.Assign) {
+          loc = DAG.Locality.THREAD_LOCAL;
         } else {
-          // Sink is not ParDo.. set null locality.
+          // cannot chain, if there is any other sink
           loc = null;
           break;
         }
       }
-    } else {
-      // Source is not ParDo... set null locality
-      loc = null;
     }
 
     streamMeta.setLocality(loc);
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
index e9e9a5b5c6a8..c5521d1c7001 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/ApexRunnerTest.java
@@ -18,15 +18,23 @@
 package org.apache.beam.runners.apex;
 
 import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
 import com.datatorrent.api.DAG.OperatorMeta;
 import com.datatorrent.stram.engine.OperatorContext;
+import com.google.common.collect.Sets;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.util.Properties;
+import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.PCollection;
+import org.hamcrest.Matchers;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -41,16 +49,13 @@ public void testConfigProperties() throws Exception {
     String operName = "testProperties";
     ApexPipelineOptions options = PipelineOptionsFactory.create()
         .as(ApexPipelineOptions.class);
-    options.setRunner(ApexRunner.class);
 
     // default configuration from class path
-    Pipeline p = Pipeline.create(options);
+    Pipeline p = Pipeline.create();
     Create.Values<Void> empty = Create.empty(VoidCoder.of());
     p.apply(operName, empty);
-    ApexRunnerResult result = (ApexRunnerResult) p.run();
-    result.cancel();
 
-    DAG dag = result.getApexDAG();
+    DAG dag = TestApexRunner.translate(p, options);
     OperatorMeta t1Meta = dag.getOperatorMeta(operName);
     Assert.assertNotNull(t1Meta);
     Assert.assertEquals(new Integer(32), t1Meta.getValue(OperatorContext.MEMORY_MB));
@@ -63,14 +68,40 @@ public void testConfigProperties() throws Exception {
       props.store(fos, "");
     }
     options.setConfigFile(tmp.getAbsolutePath());
-    result = (ApexRunnerResult) p.run();
-    result.cancel();
+    dag = TestApexRunner.translate(p, options);
     tmp.delete();
-    dag = result.getApexDAG();
+
     t1Meta = dag.getOperatorMeta(operName);
     Assert.assertNotNull(t1Meta);
     Assert.assertEquals(new Integer(64), t1Meta.getValue(OperatorContext.MEMORY_MB));
 
   }
 
+  @Test
+  public void testParDoChaining() throws Exception {
+    Pipeline p = Pipeline.create();
+    long numElements = 1000;
+    PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements));
+    PAssert.thatSingleton(input.apply("Count", Count.<Long>globally())).isEqualTo(numElements);
+
+    ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
+    DAG dag = TestApexRunner.translate(p, options);
+
+    String[] expectedThreadLocal = { "/CreateActual/FilterActuals/Window.Assign" };
+    Set<String> actualThreadLocal = Sets.newHashSet();
+    for (DAG.StreamMeta sm : dag.getAllStreamsMeta()) {
+      DAG.OutputPortMeta opm = sm.getSource();
+      if (sm.getLocality() == Locality.THREAD_LOCAL) {
+         String name = opm.getOperatorMeta().getName();
+         String prefix = "PAssert$";
+         if (name.startsWith(prefix)) {
+           // remove indeterministic prefix
+           name = name.substring(prefix.length() + 1);
+         }
+         actualThreadLocal.add(name);
+      }
+    }
+    Assert.assertThat(actualThreadLocal, Matchers.hasItems(expectedThreadLocal));
+  }
+
 }
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
index 64ca0ee4fd07..929778a529a6 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/FlattenPCollectionTranslatorTest.java
@@ -26,12 +26,10 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import org.apache.apex.api.EmbeddedAppLauncher;
-import org.apache.apex.api.Launcher;
-import org.apache.apex.api.Launcher.LaunchMode;
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.runners.apex.ApexRunner;
 import org.apache.beam.runners.apex.ApexRunnerResult;
+import org.apache.beam.runners.apex.TestApexRunner;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -100,16 +98,11 @@ public void processElement(ProcessContext c) throws Exception {
   @Test
   public void testFlattenSingleCollection() {
     ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
-    options.setRunner(ApexRunner.class);
-    ApexPipelineTranslator translator = new ApexPipelineTranslator(options);
-    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
-    DAG dag = launcher.getDAG();
-
-    Pipeline p = Pipeline.create(options);
+    Pipeline p = Pipeline.create();
     PCollection<String> single = p.apply(Create.of(Collections.singletonList("1")));
     PCollectionList.of(single).apply(Flatten.<String>pCollections())
       .apply(ParDo.of(new EmbeddedCollector()));
-    translator.translate(p, dag);
+    DAG dag = TestApexRunner.translate(p, options);
     Assert.assertNotNull(
         dag.getOperatorMeta("ParDo(EmbeddedCollector)/ParMultiDo(EmbeddedCollector)"));
   }

From 96de8d735c40ee5f823ff17966992c433d68f296 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Fri, 5 May 2017 15:23:03 -0700
Subject: [PATCH 107/387] Remove uses of context in coder size estimation
 calls.

    find . -type f -name '*.java' | xargs sed -i '' 's/\([a-zA-Z]*[bB]yteSize[a-zA-Z]*[(].*\), [^,]*[Cc]ontext[^,()]*\([(][)]\)*[)]/\1)/'

plus a one-off Python script and some manual fixups.
---
 runners/google-cloud-dataflow-java/pom.xml    |  2 +-
 .../runners/dataflow/internal/IsmFormat.java  |  8 ++--
 .../dataflow/util/RandomAccessData.java       | 11 ++---
 .../dataflow/util/RandomAccessDataTest.java   |  6 +--
 .../beam/sdk/coders/BigDecimalCoder.java      |  8 ++--
 .../sdk/coders/BigEndianIntegerCoder.java     |  4 +-
 .../beam/sdk/coders/BigEndianLongCoder.java   |  4 +-
 .../beam/sdk/coders/BigIntegerCoder.java      |  6 +--
 .../beam/sdk/coders/ByteArrayCoder.java       | 10 ++---
 .../org/apache/beam/sdk/coders/ByteCoder.java |  4 +-
 .../apache/beam/sdk/coders/DoubleCoder.java   |  4 +-
 .../apache/beam/sdk/coders/DurationCoder.java |  8 ++--
 .../apache/beam/sdk/coders/InstantCoder.java  |  8 ++--
 .../beam/sdk/coders/IterableLikeCoder.java    |  9 ++---
 .../org/apache/beam/sdk/coders/KvCoder.java   | 12 +++---
 .../beam/sdk/coders/LengthPrefixCoder.java    | 11 +++--
 .../org/apache/beam/sdk/coders/MapCoder.java  | 10 ++---
 .../apache/beam/sdk/coders/NullableCoder.java | 14 +++----
 .../beam/sdk/coders/StringDelegateCoder.java  |  1 -
 .../beam/sdk/coders/StringUtf8Coder.java      | 16 ++------
 .../beam/sdk/coders/TextualIntegerCoder.java  |  4 +-
 .../apache/beam/sdk/coders/VarIntCoder.java   |  4 +-
 .../apache/beam/sdk/coders/VarLongCoder.java  |  4 +-
 .../org/apache/beam/sdk/coders/VoidCoder.java |  4 +-
 .../beam/sdk/testing/CoderProperties.java     |  6 ++-
 .../sdk/transforms/ApproximateQuantiles.java  | 22 ++++------
 .../org/apache/beam/sdk/transforms/Count.java |  4 +-
 .../org/apache/beam/sdk/transforms/Top.java   |  9 ++---
 .../beam/sdk/transforms/join/UnionCoder.java  |  8 ++--
 .../org/apache/beam/sdk/util/BitSetCoder.java |  1 -
 .../apache/beam/sdk/util/WindowedValue.java   | 15 ++++---
 .../beam/sdk/coders/BigDecimalCoderTest.java  |  5 ++-
 .../beam/sdk/coders/BigIntegerCoderTest.java  |  5 ++-
 .../beam/sdk/coders/CoderRegistryTest.java    |  4 +-
 .../sdk/coders/LengthPrefixCoderTest.java     | 24 +++++------
 .../beam/sdk/coders/NullableCoderTest.java    | 18 ++++-----
 .../apache/beam/sdk/testing/PAssertTest.java  |  4 +-
 .../beam/sdk/transforms/CombineTest.java      |  8 ++--
 .../apache/beam/sdk/transforms/ParDoTest.java |  4 +-
 .../extensions/protobuf/ByteStringCoder.java  |  8 +---
 .../protobuf/ByteStringCoderTest.java         | 10 ++---
 .../io/gcp/bigquery/TableRowJsonCoder.java    |  4 +-
 .../beam/sdk/io/kafka/KafkaRecordCoder.java   |  4 +-
 .../org/apache/beam/sdk/io/xml/JAXBCoder.java | 40 ++++++++++---------
 44 files changed, 168 insertions(+), 207 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 09901d53202b..b57904122b54 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170505-wd-2914</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170506</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
index aed514a08c7c..00e0c54c2d5f 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
@@ -704,12 +704,12 @@ public boolean consistentWithEquals() {
     }
 
     @Override
-    public boolean isRegisterByteSizeObserverCheap(KeyPrefix value, Coder.Context context) {
+    public boolean isRegisterByteSizeObserverCheap(KeyPrefix value) {
       return true;
     }
 
     @Override
-    public long getEncodedElementByteSize(KeyPrefix value, Coder.Context context)
+    public long getEncodedElementByteSize(KeyPrefix value)
         throws Exception {
       checkNotNull(value);
       return VarInt.getLength(value.getSharedKeySize())
@@ -786,12 +786,12 @@ public boolean consistentWithEquals() {
     }
 
     @Override
-    public boolean isRegisterByteSizeObserverCheap(Footer value, Coder.Context context) {
+    public boolean isRegisterByteSizeObserverCheap(Footer value) {
       return true;
     }
 
     @Override
-    public long getEncodedElementByteSize(Footer value, Coder.Context context)
+    public long getEncodedElementByteSize(Footer value)
         throws Exception {
       return Footer.FIXED_LENGTH;
     }
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
index 4e9451511d1a..f36bd78511e2 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
@@ -96,22 +96,17 @@ public boolean consistentWithEquals() {
     }
 
     @Override
-    public boolean isRegisterByteSizeObserverCheap(
-        RandomAccessData value, Coder.Context context) {
+    public boolean isRegisterByteSizeObserverCheap(RandomAccessData value) {
       return true;
     }
 
     @Override
-    protected long getEncodedElementByteSize(RandomAccessData value, Coder.Context context)
+    protected long getEncodedElementByteSize(RandomAccessData value)
         throws Exception {
       if (value == null) {
         throw new CoderException("cannot encode a null in memory stream");
       }
-      long size = 0;
-      if (!context.isWholeStream) {
-        size += VarInt.getLength(value.size);
-      }
-      return size + value.size;
+      return VarInt.getLength(value.size) + value.size;
     }
   }
 
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java
index 042e14578b78..5a7908c5dd7f 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java
@@ -61,10 +61,8 @@ public void testCoder() throws Exception {
     CoderProperties.coderSerializable(RandomAccessDataCoder.of());
     CoderProperties.structuralValueConsistentWithEquals(
         RandomAccessDataCoder.of(), streamA, streamB);
-    assertTrue(RandomAccessDataCoder.of().isRegisterByteSizeObserverCheap(streamA, Context.NESTED));
-    assertTrue(RandomAccessDataCoder.of().isRegisterByteSizeObserverCheap(streamA, Context.OUTER));
-    assertEquals(4, RandomAccessDataCoder.of().getEncodedElementByteSize(streamA, Context.NESTED));
-    assertEquals(3, RandomAccessDataCoder.of().getEncodedElementByteSize(streamA, Context.OUTER));
+    assertTrue(RandomAccessDataCoder.of().isRegisterByteSizeObserverCheap(streamA));
+    assertEquals(4, RandomAccessDataCoder.of().getEncodedElementByteSize(streamA));
   }
 
   @Test
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
index aadf085881af..97559a9abb4f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
@@ -85,7 +85,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}, because {@link #getEncodedElementByteSize} runs in constant time.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(BigDecimal value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(BigDecimal value) {
     return true;
   }
 
@@ -97,9 +97,9 @@ public boolean isRegisterByteSizeObserverCheap(BigDecimal value, Context context
    * representation of the {@link BigInteger} that, when scaled, equals the given value.
    */
   @Override
-  protected long getEncodedElementByteSize(BigDecimal value, Context context) throws Exception {
+  protected long getEncodedElementByteSize(BigDecimal value) throws Exception {
     checkNotNull(value, String.format("cannot encode a null %s", BigDecimal.class.getSimpleName()));
-    return VAR_INT_CODER.getEncodedElementByteSize(value.scale(), context.nested())
-        + BIG_INT_CODER.getEncodedElementByteSize(value.unscaledValue(), context);
+    return VAR_INT_CODER.getEncodedElementByteSize(value.scale())
+        + BIG_INT_CODER.getEncodedElementByteSize(value.unscaledValue());
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
index c3c7a968d196..a61f09951173 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
@@ -82,7 +82,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}, because {@link #getEncodedElementByteSize} runs in constant time.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Integer value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Integer value) {
     return true;
   }
 
@@ -97,7 +97,7 @@ public TypeDescriptor<Integer> getEncodedTypeDescriptor() {
    * @return {@code 4}, the size in bytes of an integer's big endian encoding.
    */
   @Override
-  protected long getEncodedElementByteSize(Integer value, Context context)
+  protected long getEncodedElementByteSize(Integer value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null Integer");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
index 5ef487801540..868160aad1e0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
@@ -82,7 +82,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}, since {@link #getEncodedElementByteSize} returns a constant.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Long value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Long value) {
     return true;
   }
 
@@ -97,7 +97,7 @@ public TypeDescriptor<Long> getEncodedTypeDescriptor() {
    * @return {@code 8}, the byte size of a big-endian encoded {@code Long}.
    */
   @Override
-  protected long getEncodedElementByteSize(Long value, Context context)
+  protected long getEncodedElementByteSize(Long value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null Long");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
index 6d14d17ee7ce..3b038afefbe3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
@@ -75,7 +75,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}, because {@link #getEncodedElementByteSize} runs in constant time.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(BigInteger value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(BigInteger value) {
     return true;
   }
 
@@ -85,8 +85,8 @@ public boolean isRegisterByteSizeObserverCheap(BigInteger value, Context context
    * @return the size of the encoding as a byte array according to {@link ByteArrayCoder}
    */
   @Override
-  protected long getEncodedElementByteSize(BigInteger value, Context context) throws Exception {
+  protected long getEncodedElementByteSize(BigInteger value) throws Exception {
     checkNotNull(value, String.format("cannot encode a null %s", BigInteger.class.getSimpleName()));
-    return BYTE_ARRAY_CODER.getEncodedElementByteSize(value.toByteArray(), context);
+    return BYTE_ARRAY_CODER.getEncodedElementByteSize(value.toByteArray());
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
index d83d8328e73e..c9393a180046 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
@@ -126,7 +126,7 @@ public Object structuralValue(byte[] value) {
    * constant time using the {@code length} of the provided array.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(byte[] value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(byte[] value) {
     return true;
   }
 
@@ -136,15 +136,11 @@ public TypeDescriptor<byte[]> getEncodedTypeDescriptor() {
   }
 
   @Override
-  protected long getEncodedElementByteSize(byte[] value, Context context)
+  protected long getEncodedElementByteSize(byte[] value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null byte[]");
     }
-    long size = 0;
-    if (!context.isWholeStream) {
-      size += VarInt.getLength(value.length);
-    }
-    return size + value.length;
+    return VarInt.getLength(value.length) + value.length;
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
index 6e4318e5a3ff..7f449d6ae895 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
@@ -91,7 +91,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}. {@link ByteCoder#getEncodedElementByteSize} returns a constant.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Byte value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Byte value) {
     return true;
   }
 
@@ -106,7 +106,7 @@ public TypeDescriptor<Byte> getEncodedTypeDescriptor() {
    * @return {@code 1}, the byte size of a {@link Byte} encoded using Java serialization.
    */
   @Override
-  protected long getEncodedElementByteSize(Byte value, Context context)
+  protected long getEncodedElementByteSize(Byte value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot estimate size for unsupported null value");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
index 12bc5e8da15f..8eff6ba9b6fb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
@@ -93,7 +93,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}. {@link DoubleCoder#getEncodedElementByteSize} returns a constant.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Double value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Double value) {
     return true;
   }
 
@@ -108,7 +108,7 @@ public TypeDescriptor<Double> getEncodedTypeDescriptor() {
    * @return {@code 8}, the byte size of a {@link Double} encoded using Java serialization.
    */
   @Override
-  protected long getEncodedElementByteSize(Double value, Context context)
+  protected long getEncodedElementByteSize(Double value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null Double");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
index 7b49d1fe31d2..8b4ae1dde31e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
@@ -89,14 +89,14 @@ public boolean consistentWithEquals() {
    * @return {@code true}, because it is cheap to ascertain the byte size of a long.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(ReadableDuration value, Context context) {
-    return LONG_CODER.isRegisterByteSizeObserverCheap(toLong(value), context);
+  public boolean isRegisterByteSizeObserverCheap(ReadableDuration value) {
+    return LONG_CODER.isRegisterByteSizeObserverCheap(toLong(value));
   }
 
   @Override
   public void registerByteSizeObserver(
-      ReadableDuration value, ElementByteSizeObserver observer, Context context) throws Exception {
-    LONG_CODER.registerByteSizeObserver(toLong(value), observer, context);
+      ReadableDuration value, ElementByteSizeObserver observer) throws Exception {
+    LONG_CODER.registerByteSizeObserver(toLong(value), observer);
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
index 56ed12b01e95..000f40672f10 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
@@ -103,16 +103,16 @@ public boolean consistentWithEquals() {
    * @return {@code true}. The byte size for a big endian long is a constant.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Instant value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Instant value) {
     return LONG_CODER.isRegisterByteSizeObserverCheap(
-        ORDER_PRESERVING_CONVERTER.convert(value), context);
+        ORDER_PRESERVING_CONVERTER.convert(value));
   }
 
   @Override
   public void registerByteSizeObserver(
-      Instant value, ElementByteSizeObserver observer, Context context) throws Exception {
+      Instant value, ElementByteSizeObserver observer) throws Exception {
     LONG_CODER.registerByteSizeObserver(
-        ORDER_PRESERVING_CONVERTER.convert(value), observer, context);
+        ORDER_PRESERVING_CONVERTER.convert(value), observer);
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
index 52b9c34c3c92..9994b3f671d7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
@@ -170,18 +170,17 @@ public void verifyDeterministic() throws NonDeterministicException {
    */
   @Override
   public boolean isRegisterByteSizeObserverCheap(
-      IterableT iterable, Context context) {
+      IterableT iterable) {
     return iterable instanceof ElementByteSizeObservableIterable;
   }
 
   @Override
   public void registerByteSizeObserver(
-      IterableT iterable, ElementByteSizeObserver observer, Context context)
+      IterableT iterable, ElementByteSizeObserver observer)
       throws Exception {
     if (iterable == null) {
       throw new CoderException("cannot encode a null Iterable");
     }
-    Context nestedContext = context.nested();
 
     if (iterable instanceof ElementByteSizeObservableIterable) {
       observer.setLazy();
@@ -196,7 +195,7 @@ public void registerByteSizeObserver(
         Collection<T> collection = (Collection<T>) iterable;
         observer.update(4L);
         for (T elem : collection) {
-          elementCoder.registerByteSizeObserver(elem, observer, nestedContext);
+          elementCoder.registerByteSizeObserver(elem, observer);
         }
       } else {
         // TODO: (BEAM-1537) Update to use an accurate count depending on size and count,
@@ -208,7 +207,7 @@ public void registerByteSizeObserver(
         long count = 0;
         for (T elem : iterable) {
           count += 1;
-          elementCoder.registerByteSizeObserver(elem, observer, nestedContext);
+          elementCoder.registerByteSizeObserver(elem, observer);
         }
         if (count > 0) {
           // Update the length based upon the number of counted elements, this helps
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
index da7f03c7599e..1df446075f83 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
@@ -105,9 +105,9 @@ public Object structuralValue(KV<K, V> kv) {
    * Returns whether both keyCoder and valueCoder are considered not expensive.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(KV<K, V> kv, Context context) {
-    return keyCoder.isRegisterByteSizeObserverCheap(kv.getKey(), context.nested())
-        && valueCoder.isRegisterByteSizeObserverCheap(kv.getValue(), context);
+  public boolean isRegisterByteSizeObserverCheap(KV<K, V> kv) {
+    return keyCoder.isRegisterByteSizeObserverCheap(kv.getKey())
+        && valueCoder.isRegisterByteSizeObserverCheap(kv.getValue());
   }
 
   /**
@@ -116,13 +116,13 @@ public boolean isRegisterByteSizeObserverCheap(KV<K, V> kv, Context context) {
    */
   @Override
   public void registerByteSizeObserver(
-      KV<K, V> kv, ElementByteSizeObserver observer, Context context)
+      KV<K, V> kv, ElementByteSizeObserver observer)
       throws Exception {
     if (kv == null) {
       throw new CoderException("cannot encode a null KV");
     }
-    keyCoder.registerByteSizeObserver(kv.getKey(), observer, context.nested());
-    valueCoder.registerByteSizeObserver(kv.getValue(), observer, context);
+    keyCoder.registerByteSizeObserver(kv.getKey(), observer);
+    valueCoder.registerByteSizeObserver(kv.getValue(), observer);
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
index 685e766eebb8..7dd2a32098a7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
@@ -107,18 +107,17 @@ public boolean consistentWithEquals() {
    * {@inheritDoc}
    */
   @Override
-  protected long getEncodedElementByteSize(T value, Context context) throws Exception {
+  protected long getEncodedElementByteSize(T value) throws Exception {
     if (valueCoder instanceof StructuredCoder) {
       // If valueCoder is a StructuredCoder then we can ask it directly for the encoded size of
       // the value, adding the number of bytes to represent the length.
-      long valueSize = ((StructuredCoder<T>) valueCoder).getEncodedElementByteSize(
-          value, Context.OUTER);
+      long valueSize = ((StructuredCoder<T>) valueCoder).getEncodedElementByteSize(value);
       return VarInt.getLength(valueSize) + valueSize;
     }
 
     // If value is not a StructuredCoder then fall back to the default StructuredCoder behavior
     // of encoding and counting the bytes. The encoding will include the length prefix.
-    return super.getEncodedElementByteSize(value, context);
+    return super.getEncodedElementByteSize(value);
   }
 
   /**
@@ -127,7 +126,7 @@ protected long getEncodedElementByteSize(T value, Context context) throws Except
    * {@inheritDoc}
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(@Nullable T value, Context context) {
-    return valueCoder.isRegisterByteSizeObserverCheap(value, Context.OUTER);
+  public boolean isRegisterByteSizeObserverCheap(@Nullable T value) {
+    return valueCoder.isRegisterByteSizeObserverCheap(value);
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
index 9e3c76829f88..7df9ca96bf38 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
@@ -146,7 +146,7 @@ public void verifyDeterministic() throws NonDeterministicException {
 
   @Override
   public void registerByteSizeObserver(
-      Map<K, V> map, ElementByteSizeObserver observer, Context context)
+      Map<K, V> map, ElementByteSizeObserver observer)
       throws Exception {
     observer.update(4L);
     if (map.isEmpty()) {
@@ -155,12 +155,12 @@ public void registerByteSizeObserver(
     Iterator<Entry<K, V>> entries = map.entrySet().iterator();
     Entry<K, V> entry = entries.next();
     while (entries.hasNext()) {
-      keyCoder.registerByteSizeObserver(entry.getKey(), observer, context.nested());
-      valueCoder.registerByteSizeObserver(entry.getValue(), observer, context.nested());
+      keyCoder.registerByteSizeObserver(entry.getKey(), observer);
+      valueCoder.registerByteSizeObserver(entry.getValue(), observer);
       entry = entries.next();
     }
-    keyCoder.registerByteSizeObserver(entry.getKey(), observer, context.nested());
-    valueCoder.registerByteSizeObserver(entry.getValue(), observer, context);
+    keyCoder.registerByteSizeObserver(entry.getKey(), observer);
+    valueCoder.registerByteSizeObserver(entry.getValue(), observer);
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
index d1eea9aa2fe4..e46591e4c799 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
@@ -127,10 +127,10 @@ public Object structuralValue(@Nullable T value) {
    */
   @Override
   public void registerByteSizeObserver(
-      @Nullable T value, ElementByteSizeObserver observer, Context context) throws Exception {
+      @Nullable T value, ElementByteSizeObserver observer) throws Exception {
     observer.update(1);
     if (value != null) {
-      valueCoder.registerByteSizeObserver(value, observer, context);
+      valueCoder.registerByteSizeObserver(value, observer);
     }
   }
 
@@ -142,7 +142,7 @@ public void registerByteSizeObserver(
    * {@inheritDoc}
    */
   @Override
-  protected long getEncodedElementByteSize(@Nullable T value, Context context) throws Exception {
+  protected long getEncodedElementByteSize(@Nullable T value) throws Exception {
     if (value == null) {
       return 1;
     }
@@ -151,12 +151,12 @@ protected long getEncodedElementByteSize(@Nullable T value, Context context) thr
       // If valueCoder is a StructuredCoder then we can ask it directly for the encoded size of
       // the value, adding 1 byte to count the null indicator.
       return 1  + ((StructuredCoder<T>) valueCoder)
-          .getEncodedElementByteSize(value, context);
+          .getEncodedElementByteSize(value);
     }
 
     // If value is not a StructuredCoder then fall back to the default StructuredCoder behavior
     // of encoding and counting the bytes. The encoding will include the null indicator byte.
-    return super.getEncodedElementByteSize(value, context);
+    return super.getEncodedElementByteSize(value);
   }
 
   /**
@@ -165,11 +165,11 @@ protected long getEncodedElementByteSize(@Nullable T value, Context context) thr
    * {@inheritDoc}
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(@Nullable T value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(@Nullable T value) {
     if (value == null) {
       return true;
     }
-    return valueCoder.isRegisterByteSizeObserverCheap(value, context);
+    return valueCoder.isRegisterByteSizeObserverCheap(value);
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java
index 39a16587cc94..1f4538fc1eff 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java
@@ -125,4 +125,3 @@ public TypeDescriptor<T> getEncodedTypeDescriptor() {
     return delegateCoder.getEncodedTypeDescriptor();
   }
 }
-
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
index 42931ca60671..44856e8a045f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
@@ -18,8 +18,6 @@
 package org.apache.beam.sdk.coders;
 
 import com.google.common.base.Utf8;
-import com.google.common.io.ByteStreams;
-import com.google.common.io.CountingOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.EOFException;
@@ -128,20 +126,12 @@ public TypeDescriptor<String> getEncodedTypeDescriptor() {
    * the byte size of the encoding plus the encoded length prefix.
    */
   @Override
-  public long getEncodedElementByteSize(String value, Context context)
+  public long getEncodedElementByteSize(String value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null String");
     }
-    if (context.isWholeStream) {
-      return Utf8.encodedLength(value);
-    } else {
-      try (CountingOutputStream countingStream =
-          new CountingOutputStream(ByteStreams.nullOutputStream())) {
-        DataOutputStream stream = new DataOutputStream(countingStream);
-        writeString(value, stream);
-        return countingStream.getCount();
-      }
-    }
+    int size = Utf8.encodedLength(value);
+    return VarInt.getLength(size) + size;
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
index 9743c4c3f102..718811c9061b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
@@ -70,11 +70,11 @@ public TypeDescriptor<Integer> getEncodedTypeDescriptor() {
   }
 
   @Override
-  protected long getEncodedElementByteSize(Integer value, Context context) throws Exception {
+  protected long getEncodedElementByteSize(Integer value) throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null Integer");
     }
     String textualValue = value.toString();
-    return StringUtf8Coder.of().getEncodedElementByteSize(textualValue, context);
+    return StringUtf8Coder.of().getEncodedElementByteSize(textualValue);
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
index 30f9c0977a6c..bda66bbeb40d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
@@ -83,7 +83,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}. {@link #getEncodedElementByteSize} is cheap.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Integer value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Integer value) {
     return true;
   }
 
@@ -93,7 +93,7 @@ public TypeDescriptor<Integer> getEncodedTypeDescriptor() {
   }
 
   @Override
-  protected long getEncodedElementByteSize(Integer value, Context context)
+  protected long getEncodedElementByteSize(Integer value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null Integer");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
index 9a7b125cbca1..bf651c37bb6f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
@@ -89,7 +89,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}. {@link #getEncodedElementByteSize} is cheap.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Long value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Long value) {
     return true;
   }
 
@@ -99,7 +99,7 @@ public TypeDescriptor<Long> getEncodedTypeDescriptor() {
   }
 
   @Override
-  protected long getEncodedElementByteSize(Long value, Context context)
+  protected long getEncodedElementByteSize(Long value)
       throws Exception {
     if (value == null) {
       throw new CoderException("cannot encode a null Long");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
index 829bd2020605..4467faa84f62 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
@@ -67,7 +67,7 @@ public boolean consistentWithEquals() {
    * @return {@code true}. {@link VoidCoder#getEncodedElementByteSize} runs in constant time.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(Void value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(Void value) {
     return true;
   }
 
@@ -77,7 +77,7 @@ public TypeDescriptor<Void> getEncodedTypeDescriptor() {
   }
 
   @Override
-  protected long getEncodedElementByteSize(Void value, Context context)
+  protected long getEncodedElementByteSize(Void value)
       throws Exception {
     return 0;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java
index 6e0e2645eac7..f660f7d0d810 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java
@@ -397,13 +397,15 @@ public static <T> void testByteCount(Coder<T> coder, Coder.Context context, T[]
 
     try (CountingOutputStream os = new CountingOutputStream(ByteStreams.nullOutputStream())) {
       for (T elem : elements) {
-        coder.registerByteSizeObserver(elem, observer, context);
+        coder.registerByteSizeObserver(elem, observer);
         coder.encode(elem, os, context);
         observer.advance();
       }
       long expectedLength = os.getCount();
 
-      assertEquals(expectedLength, observer.getSum());
+      if (!context.isWholeStream) {
+        assertEquals(expectedLength, observer.getSum());
+      }
       assertEquals(elements.length, observer.getCount());
     }
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
index b05f2233156a..37d5a551ccbf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
@@ -741,24 +741,16 @@ private QuantileBuffer<T> decodeBuffer(
     @Override
     public void registerByteSizeObserver(
         QuantileState<T, ComparatorT> state,
-        ElementByteSizeObserver observer,
-        Coder.Context context)
+        ElementByteSizeObserver observer)
         throws Exception {
-      Coder.Context nestedContext = context.nested();
-      elementCoder.registerByteSizeObserver(
-          state.min, observer, nestedContext);
-      elementCoder.registerByteSizeObserver(
-          state.max, observer, nestedContext);
-      elementListCoder.registerByteSizeObserver(
-          state.unbufferedElements, observer, nestedContext);
-
-      BigEndianIntegerCoder.of().registerByteSizeObserver(
-          state.buffers.size(), observer, nestedContext);
+      elementCoder.registerByteSizeObserver(state.min, observer);
+      elementCoder.registerByteSizeObserver(state.max, observer);
+      elementListCoder.registerByteSizeObserver(state.unbufferedElements, observer);
+
+      BigEndianIntegerCoder.of().registerByteSizeObserver(state.buffers.size(), observer);
       for (QuantileBuffer<T> buffer : state.buffers) {
         observer.update(4L + 8);
-
-        elementListCoder.registerByteSizeObserver(
-            buffer.elements, observer, nestedContext);
+        elementListCoder.registerByteSizeObserver(buffer.elements, observer);
       }
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
index 753e14c84c96..497d62b2f4a1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
@@ -185,12 +185,12 @@ public long[] decode(InputStream inStream, Context context)
         }
 
         @Override
-        public boolean isRegisterByteSizeObserverCheap(long[] value, Context context) {
+        public boolean isRegisterByteSizeObserverCheap(long[] value) {
           return true;
         }
 
         @Override
-        protected long getEncodedElementByteSize(long[] value, Context context) {
+        protected long getEncodedElementByteSize(long[] value) {
           return VarInt.getLength(value[0]);
         }
       };
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
index 9d5db74a7c52..c0381a71ee5d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
@@ -557,16 +557,15 @@ public void verifyDeterministic() throws NonDeterministicException {
 
     @Override
     public boolean isRegisterByteSizeObserverCheap(
-        BoundedHeap<T, ComparatorT> value, Context context) {
-      return listCoder.isRegisterByteSizeObserverCheap(
-          value.asList(), context);
+        BoundedHeap<T, ComparatorT> value) {
+      return listCoder.isRegisterByteSizeObserverCheap(value.asList());
     }
 
     @Override
     public void registerByteSizeObserver(
-        BoundedHeap<T, ComparatorT> value, ElementByteSizeObserver observer, Context context)
+        BoundedHeap<T, ComparatorT> value, ElementByteSizeObserver observer)
             throws Exception {
-      listCoder.registerByteSizeObserver(value.asList(), observer, context);
+      listCoder.registerByteSizeObserver(value.asList(), observer);
     }
 
     @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
index 4a2a2867bd22..3194a37f4966 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
@@ -100,11 +100,11 @@ public List<? extends Coder<?>> getElementCoders() {
    * time, we defer the return value to that coder.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(RawUnionValue union, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(RawUnionValue union) {
     int index = getIndexForEncoding(union);
     @SuppressWarnings("unchecked")
     Coder<Object> coder = (Coder<Object>) elementCoders.get(index);
-    return coder.isRegisterByteSizeObserverCheap(union.getValue(), context);
+    return coder.isRegisterByteSizeObserverCheap(union.getValue());
   }
 
   /**
@@ -112,7 +112,7 @@ public boolean isRegisterByteSizeObserverCheap(RawUnionValue union, Context cont
    */
   @Override
   public void registerByteSizeObserver(
-      RawUnionValue union, ElementByteSizeObserver observer, Context context)
+      RawUnionValue union, ElementByteSizeObserver observer)
       throws Exception {
     int index = getIndexForEncoding(union);
     // Write out the union tag.
@@ -120,7 +120,7 @@ public void registerByteSizeObserver(
     // Write out the actual value.
     @SuppressWarnings("unchecked")
     Coder<Object> coder = (Coder<Object>) elementCoders.get(index);
-    coder.registerByteSizeObserver(union.getValue(), observer, context);
+    coder.registerByteSizeObserver(union.getValue(), observer);
   }
 
   /////////////////////////////////////////////////////////////////////////////
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
index b646bf6b7107..a0896f53cfd7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
@@ -59,4 +59,3 @@ public void verifyDeterministic() throws NonDeterministicException {
     BYTE_ARRAY_CODER.verifyDeterministic();
   }
 }
-
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
index 1e725500e5ff..1b7e335deacf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
@@ -670,12 +670,11 @@ public void verifyDeterministic() throws NonDeterministicException {
 
     @Override
     public void registerByteSizeObserver(WindowedValue<T> value,
-                                         ElementByteSizeObserver observer,
-                                         Context context) throws Exception {
-      InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer, context.nested());
-      windowsCoder.registerByteSizeObserver(value.getWindows(), observer, context.nested());
-      PaneInfoCoder.INSTANCE.registerByteSizeObserver(value.getPane(), observer, context.nested());
-      valueCoder.registerByteSizeObserver(value.getValue(), observer, context);
+                                         ElementByteSizeObserver observer) throws Exception {
+      InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer);
+      windowsCoder.registerByteSizeObserver(value.getWindows(), observer);
+      PaneInfoCoder.INSTANCE.registerByteSizeObserver(value.getPane(), observer);
+      valueCoder.registerByteSizeObserver(value.getValue(), observer);
     }
 
     @Override
@@ -733,9 +732,9 @@ public void verifyDeterministic() throws NonDeterministicException {
 
     @Override
     public void registerByteSizeObserver(
-        WindowedValue<T> value, ElementByteSizeObserver observer, Context context)
+        WindowedValue<T> value, ElementByteSizeObserver observer)
         throws Exception {
-      valueCoder.registerByteSizeObserver(value.getValue(), observer, context);
+      valueCoder.registerByteSizeObserver(value.getValue(), observer);
     }
 
     @Override
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java
index 8aa260453c73..a8a1bc872678 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java
@@ -100,10 +100,11 @@ public void testWireFormatEncode() throws Exception {
   public void testGetEncodedElementByteSize() throws Exception {
     TestElementByteSizeObserver observer = new TestElementByteSizeObserver();
     for (BigDecimal value : TEST_VALUES) {
-      TEST_CODER.registerByteSizeObserver(value, observer, Coder.Context.OUTER);
+      TEST_CODER.registerByteSizeObserver(value, observer);
       observer.advance();
       assertThat(observer.getSumAndReset(),
-          equalTo((long) CoderUtils.encodeToByteArray(TEST_CODER, value).length));
+          equalTo((long) CoderUtils.encodeToByteArray(
+              TEST_CODER, value, Coder.Context.NESTED).length));
     }
   }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java
index e2fd012e1275..bfd6b4ad0c9c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java
@@ -75,11 +75,12 @@ public void testWireFormatEncode() throws Exception {
   public void testGetEncodedElementByteSize() throws Exception {
     TestElementByteSizeObserver observer = new TestElementByteSizeObserver();
     for (BigInteger value : TEST_VALUES) {
-      TEST_CODER.registerByteSizeObserver(value, observer, Coder.Context.OUTER);
+      TEST_CODER.registerByteSizeObserver(value, observer);
       observer.advance();
       assertThat(
           observer.getSumAndReset(),
-          equalTo((long) CoderUtils.encodeToByteArray(TEST_CODER, value).length));
+          equalTo((long) CoderUtils.encodeToByteArray(
+              TEST_CODER, value, Coder.Context.NESTED).length));
     }
   }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
index 9568324ad3b4..7ca7fb993717 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
@@ -399,13 +399,13 @@ public Object structuralValue(MyValue value) {
     }
 
     @Override
-    public boolean isRegisterByteSizeObserverCheap(MyValue value, Context context) {
+    public boolean isRegisterByteSizeObserverCheap(MyValue value) {
       return true;
     }
 
     @Override
     public void registerByteSizeObserver(
-        MyValue value, ElementByteSizeObserver observer, Context context)
+        MyValue value, ElementByteSizeObserver observer)
         throws Exception {
       observer.update(0L);
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java
index fa81a7c22990..9a09b8670ae5 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java
@@ -64,23 +64,22 @@ public void testCoderIsSerializableWithWellKnownCoderType() throws Exception {
 
   @Test
   public void testEncodedSize() throws Exception {
-    assertEquals(4L,
-        TEST_CODER.getEncodedElementByteSize(TEST_VALUES.get(0), Coder.Context.NESTED));
-    assertEquals(4L,
-        TEST_CODER.getEncodedElementByteSize(TEST_VALUES.get(0), Coder.Context.OUTER));
+    assertEquals(5L,
+        TEST_CODER.getEncodedElementByteSize(TEST_VALUES.get(0)));
   }
 
   @Test
   public void testObserverIsCheap() throws Exception {
-    NullableCoder<Double> coder = NullableCoder.of(DoubleCoder.of());
-    assertTrue(coder.isRegisterByteSizeObserverCheap(5.0, Coder.Context.OUTER));
+    LengthPrefixCoder<Double> coder = LengthPrefixCoder.of(DoubleCoder.of());
+    assertTrue(coder.isRegisterByteSizeObserverCheap(5.0));
   }
 
   @Test
   public void testObserverIsNotCheap() throws Exception {
-    NullableCoder<List<String>> coder = NullableCoder.of(ListCoder.of(StringUtf8Coder.of()));
+    LengthPrefixCoder<List<String>> coder =
+        LengthPrefixCoder.of(ListCoder.of(StringUtf8Coder.of()));
     assertFalse(coder.isRegisterByteSizeObserverCheap(
-        ImmutableList.of("hi", "test"), Coder.Context.OUTER));
+        ImmutableList.of("hi", "test")));
   }
 
   @Test
@@ -92,11 +91,10 @@ public void testDecodeEncodeEquals() throws Exception {
 
   @Test
   public void testRegisterByteSizeObserver() throws Exception {
-    CoderProperties.testByteCount(TEST_CODER, Coder.Context.OUTER,
-        new byte[][]{{ 0xa, 0xb, 0xc }});
-
-    CoderProperties.testByteCount(TEST_CODER, Coder.Context.NESTED,
-        new byte[][]{{ 0xa, 0xb, 0xc }, {}, {}, { 0xd, 0xe }, {}});
+    CoderProperties.testByteCount(
+        LengthPrefixCoder.of(VarIntCoder.of()),
+        Coder.Context.NESTED,
+        new Integer[]{0, 10, 1000});
   }
 
   @Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
index c0a4bed84a42..d6d7de8b5779 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
@@ -98,38 +98,34 @@ public void testWireFormatEncode() throws Exception {
   @Test
   public void testEncodedSize() throws Exception {
     NullableCoder<Double> coder = NullableCoder.of(DoubleCoder.of());
-    assertEquals(1, coder.getEncodedElementByteSize(null, Coder.Context.OUTER));
-    assertEquals(9, coder.getEncodedElementByteSize(5.0, Coder.Context.OUTER));
+    assertEquals(1, coder.getEncodedElementByteSize(null));
+    assertEquals(9, coder.getEncodedElementByteSize(5.0));
   }
 
   @Test
   public void testEncodedSizeNested() throws Exception {
     NullableCoder<String> varLenCoder = NullableCoder.of(StringUtf8Coder.of());
-
-    assertEquals(1, varLenCoder.getEncodedElementByteSize(null, Context.OUTER));
-    assertEquals(1, varLenCoder.getEncodedElementByteSize(null, Context.NESTED));
-
-    assertEquals(5, varLenCoder.getEncodedElementByteSize("spam", Context.OUTER));
-    assertEquals(6, varLenCoder.getEncodedElementByteSize("spam", Context.NESTED));
+    assertEquals(1, varLenCoder.getEncodedElementByteSize(null));
+    assertEquals(6, varLenCoder.getEncodedElementByteSize("spam"));
   }
 
   @Test
   public void testObserverIsCheap() throws Exception {
     NullableCoder<Double> coder = NullableCoder.of(DoubleCoder.of());
-    assertTrue(coder.isRegisterByteSizeObserverCheap(5.0, Coder.Context.OUTER));
+    assertTrue(coder.isRegisterByteSizeObserverCheap(5.0));
   }
 
   @Test
   public void testObserverIsNotCheap() throws Exception {
     NullableCoder<List<String>> coder = NullableCoder.of(ListCoder.of(StringUtf8Coder.of()));
     assertFalse(coder.isRegisterByteSizeObserverCheap(
-        ImmutableList.of("hi", "test"), Coder.Context.OUTER));
+        ImmutableList.of("hi", "test")));
   }
 
   @Test
   public void testObserverIsAlwaysCheapForNullValues() throws Exception {
     NullableCoder<List<String>> coder = NullableCoder.of(ListCoder.of(StringUtf8Coder.of()));
-    assertTrue(coder.isRegisterByteSizeObserverCheap(null, Coder.Context.OUTER));
+    assertTrue(coder.isRegisterByteSizeObserverCheap(null));
   }
 
   @Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
index 2ef892c0e822..83f348c2678f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
@@ -109,13 +109,13 @@ public NotSerializableObject decode(InputStream inStream, Context context)
     }
 
     @Override
-    public boolean isRegisterByteSizeObserverCheap(NotSerializableObject value, Context context) {
+    public boolean isRegisterByteSizeObserverCheap(NotSerializableObject value) {
       return true;
     }
 
     @Override
     public void registerByteSizeObserver(
-        NotSerializableObject value, ElementByteSizeObserver observer, Context context)
+        NotSerializableObject value, ElementByteSizeObserver observer)
         throws Exception {
       observer.update(0L);
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
index 12619e03f8ac..a70af94f3768 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
@@ -895,16 +895,16 @@ public void verifyDeterministic() throws NonDeterministicException { }
 
       @Override
       public boolean isRegisterByteSizeObserverCheap(
-          CountSum value, Context context) {
+          CountSum value) {
         return true;
       }
 
       @Override
       public void registerByteSizeObserver(
-          CountSum value, ElementByteSizeObserver observer, Context context)
+          CountSum value, ElementByteSizeObserver observer)
           throws Exception {
-        LONG_CODER.registerByteSizeObserver(value.count, observer, context.nested());
-        DOUBLE_CODER.registerByteSizeObserver(value.sum, observer, context);
+        LONG_CODER.registerByteSizeObserver(value.count, observer);
+        DOUBLE_CODER.registerByteSizeObserver(value.sum, observer);
       }
     }
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index cbbe7f13963c..d2cb980f3a75 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -997,13 +997,13 @@ public TestDummy decode(InputStream inStream, Context context)
     }
 
     @Override
-    public boolean isRegisterByteSizeObserverCheap(TestDummy value, Context context) {
+    public boolean isRegisterByteSizeObserverCheap(TestDummy value) {
       return true;
     }
 
     @Override
     public void registerByteSizeObserver(
-        TestDummy value, ElementByteSizeObserver observer, Context context)
+        TestDummy value, ElementByteSizeObserver observer)
         throws Exception {
       observer.update(0L);
     }
diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
index 0781cf1fdac8..325c69dfee8e 100644
--- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
+++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
@@ -76,12 +76,8 @@ public ByteString decode(InputStream inStream, Context context) throws IOExcepti
   }
 
   @Override
-  protected long getEncodedElementByteSize(ByteString value, Context context) throws Exception {
+  protected long getEncodedElementByteSize(ByteString value) throws Exception {
     int size = value.size();
-
-    if (context.isWholeStream) {
-      return size;
-    }
     return VarInt.getLength(size) + size;
   }
 
@@ -106,7 +102,7 @@ public boolean consistentWithEquals() {
    * <p>Returns true. {@link ByteString#size} returns the size of an array and a {@link VarInt}.
    */
   @Override
-  public boolean isRegisterByteSizeObserverCheap(ByteString value, Context context) {
+  public boolean isRegisterByteSizeObserverCheap(ByteString value) {
     return true;
   }
 
diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java
index 8fdb851103d0..d1800a6457d5 100644
--- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java
+++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java
@@ -115,12 +115,10 @@ public void testNestedCoding() throws Throwable {
   }
 
   @Test
-  public void testEncodedElementByteSizeInAllContexts() throws Throwable {
-    for (Context context : CoderProperties.ALL_CONTEXTS) {
-      for (ByteString value : TEST_VALUES) {
-        byte[] encoded = CoderUtils.encodeToByteArray(TEST_CODER, value, context);
-        assertEquals(encoded.length, TEST_CODER.getEncodedElementByteSize(value, context));
-      }
+  public void testEncodedElementByteSize() throws Throwable {
+    for (ByteString value : TEST_VALUES) {
+      byte[] encoded = CoderUtils.encodeToByteArray(TEST_CODER, value, Context.NESTED);
+      assertEquals(encoded.length, TEST_CODER.getEncodedElementByteSize(value));
     }
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
index 7ca8958e01de..cfec991d7a3a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
@@ -52,10 +52,10 @@ public TableRow decode(InputStream inStream, Context context)
   }
 
   @Override
-  protected long getEncodedElementByteSize(TableRow value, Context context)
+  protected long getEncodedElementByteSize(TableRow value)
       throws Exception {
     String strValue = MAPPER.writeValueAsString(value);
-    return StringUtf8Coder.of().getEncodedElementByteSize(strValue, context);
+    return StringUtf8Coder.of().getEncodedElementByteSize(strValue);
   }
 
   /////////////////////////////////////////////////////////////////////////////
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
index 0f5dc4cd19f5..d838a0d7943f 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
@@ -83,8 +83,8 @@ public void verifyDeterministic() throws NonDeterministicException {
   }
 
   @Override
-  public boolean isRegisterByteSizeObserverCheap(KafkaRecord<K, V> value, Context context) {
-    return kvCoder.isRegisterByteSizeObserverCheap(value.getKV(), context);
+  public boolean isRegisterByteSizeObserverCheap(KafkaRecord<K, V> value) {
+    return kvCoder.isRegisterByteSizeObserverCheap(value.getKV());
     //TODO : do we have to implement getEncodedSize()?
   }
 
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
index 75a461924907..5b2ec0251484 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
@@ -18,6 +18,7 @@
 package org.apache.beam.sdk.io.xml;
 
 import com.google.common.io.ByteStreams;
+import java.io.ByteArrayOutputStream;
 import java.io.FilterInputStream;
 import java.io.FilterOutputStream;
 import java.io.IOException;
@@ -87,37 +88,40 @@ public static <T> JAXBCoder<T> of(Class<T> jaxbClass) {
   }
 
   @Override
-  public void encode(T value, OutputStream outStream, Context context)
-      throws CoderException, IOException {
+  public void encode(T value, OutputStream outStream) throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
     try {
-      if (!context.isWholeStream) {
-        try {
-          long size = getEncodedElementByteSize(value, Context.OUTER);
-          // record the number of bytes the XML consists of so when reading we only read the encoded
-          // value
-          VarInt.encode(size, outStream);
-        } catch (Exception e) {
-          throw new CoderException(
-              "An Exception occured while trying to get the size of an encoded representation", e);
-        }
-      }
-
-      jaxbMarshaller.get().marshal(value, new CloseIgnoringOutputStream(outStream));
+      jaxbMarshaller.get().marshal(value, baos);
     } catch (JAXBException e) {
       throw new CoderException(e);
     }
+    VarInt.encode(baos.size(), outStream);
+    baos.writeTo(outStream);
+  }
+
+  @Override
+  public void encode(T value, OutputStream outStream, Context context)
+      throws CoderException, IOException {
+    if (context.isWholeStream) {
+      try {
+        jaxbMarshaller.get().marshal(value, new CloseIgnoringOutputStream(outStream));
+      } catch (JAXBException e) {
+        throw new CoderException(e);
+      }
+    } else {
+      encode(value, outStream);
+    }
   }
 
   @Override
   public T decode(InputStream inStream, Context context) throws CoderException, IOException {
     try {
-      InputStream stream = inStream;
       if (!context.isWholeStream) {
         long limit = VarInt.decodeLong(inStream);
-        stream = ByteStreams.limit(inStream, limit);
+        inStream = ByteStreams.limit(inStream, limit);
       }
       @SuppressWarnings("unchecked")
-      T obj = (T) jaxbUnmarshaller.get().unmarshal(new CloseIgnoringInputStream(stream));
+      T obj = (T) jaxbUnmarshaller.get().unmarshal(new CloseIgnoringInputStream(inStream));
       return obj;
     } catch (JAXBException e) {
       throw new CoderException(e);

From 938cb2f4c2c5b47ac9cd5742f23ede9a88da2b66 Mon Sep 17 00:00:00 2001
From: Pablo <pabloem@google.com>
Date: Sun, 7 May 2017 20:29:30 -0700
Subject: [PATCH 108/387] Fixing typo in datastore_wordcount.py

---
 .../python/apache_beam/examples/cookbook/datastore_wordcount.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
index d694cdace600..1f88398fbbad 100644
--- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
+++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
@@ -108,7 +108,7 @@ def process(self, element):
     words = re.findall(r'[A-Za-z\']+', text_line)
     for w in words:
       self.word_length_counter.inc(len(w))
-      self.word_len_dis.update(len(w))
+      self.word_lengths_dist.update(len(w))
       self.word_counter.inc()
     return words
 

From 385cd5d52fb74025ce3532961340fb72b9ca2b53 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Sun, 7 May 2017 18:08:49 -0700
Subject: [PATCH 109/387] Only cythonize files within apache_beam

---
 sdks/python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index f527362aaa07..681abbfce96b 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -124,7 +124,7 @@ def get_version():
     package_data={'apache_beam': [
         '*/*.pyx', '*/*/*.pyx', '*/*.pxd', '*/*/*.pxd', 'tests/data/*']},
     ext_modules=cythonize([
-        '**/*.pyx',
+        'apache_beam/**/*.pyx',
         'apache_beam/coders/coder_impl.py',
         'apache_beam/metrics/execution.py',
         'apache_beam/runners/common.py',

From 8ef9861fb419e34978593d961eed186f5ef87406 Mon Sep 17 00:00:00 2001
From: JingsongLi <lzljs3620320@aliyun.com>
Date: Wed, 19 Apr 2017 19:42:59 +0800
Subject: [PATCH 110/387] [BEAM-1723] deduplication of UnboundedSource in Flink
 runner

---
 .../FlinkStreamingTransformTranslators.java   |  54 ++++-
 .../streaming/io/DedupingOperator.java        | 187 ++++++++++++++++++
 .../streaming/io/UnboundedSourceWrapper.java  |  15 +-
 .../flink/streaming/DedupingOperatorTest.java | 131 ++++++++++++
 .../streaming/UnboundedSourceWrapperTest.java |  29 +--
 5 files changed, 393 insertions(+), 23 deletions(-)
 create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/DedupingOperator.java
 create mode 100644 runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DedupingOperatorTest.java

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index 615eaea29735..9a9320564739 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -44,6 +44,7 @@
 import org.apache.beam.runners.flink.translation.wrappers.streaming.WindowDoFnOperator;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.WorkItemKeySelector;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.io.BoundedSourceWrapper;
+import org.apache.beam.runners.flink.translation.wrappers.streaming.io.DedupingOperator;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.io.UnboundedSourceWrapper;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
@@ -73,12 +74,16 @@
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.ValueWithRecordId;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.common.functions.RichFlatMapFunction;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.streaming.api.collector.selector.OutputSelector;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.DataStreamSource;
@@ -148,20 +153,37 @@ public void translateNode(
         FlinkStreamingTranslationContext context) {
       PCollection<T> output = context.getOutput(transform);
 
+      DataStream<WindowedValue<T>> source;
+      DataStream<WindowedValue<ValueWithRecordId<T>>> nonDedupSource;
       TypeInformation<WindowedValue<T>> outputTypeInfo =
           context.getTypeInfo(context.getOutput(transform));
 
-      DataStream<WindowedValue<T>> source;
+      Coder<T> coder = context.getOutput(transform).getCoder();
+
+      TypeInformation<WindowedValue<ValueWithRecordId<T>>> withIdTypeInfo =
+          new CoderTypeInformation<>(WindowedValue.getFullCoder(
+              ValueWithRecordId.ValueWithRecordIdCoder.of(coder),
+              output.getWindowingStrategy().getWindowFn().windowCoder()));
+
       try {
+
         UnboundedSourceWrapper<T, ?> sourceWrapper =
             new UnboundedSourceWrapper<>(
                 context.getCurrentTransform().getFullName(),
                 context.getPipelineOptions(),
                 transform.getSource(),
                 context.getExecutionEnvironment().getParallelism());
-        source = context
+        nonDedupSource = context
             .getExecutionEnvironment()
-            .addSource(sourceWrapper).name(transform.getName()).returns(outputTypeInfo);
+            .addSource(sourceWrapper).name(transform.getName()).returns(withIdTypeInfo);
+
+        if (transform.getSource().requiresDeduping()) {
+          source = nonDedupSource.keyBy(
+              new ValueWithRecordIdKeySelector<T>())
+              .transform("debuping", outputTypeInfo, new DedupingOperator<T>());
+        } else {
+          source = nonDedupSource.flatMap(new StripIdsMap<T>());
+        }
       } catch (Exception e) {
         throw new RuntimeException(
             "Error while translating UnboundedSource: " + transform.getSource(), e);
@@ -171,6 +193,32 @@ public void translateNode(
     }
   }
 
+  private static class ValueWithRecordIdKeySelector<T>
+      implements KeySelector<WindowedValue<ValueWithRecordId<T>>, ByteBuffer>,
+      ResultTypeQueryable<ByteBuffer> {
+
+    @Override
+    public ByteBuffer getKey(WindowedValue<ValueWithRecordId<T>> value) throws Exception {
+      return ByteBuffer.wrap(value.getValue().getId());
+    }
+
+    @Override
+    public TypeInformation<ByteBuffer> getProducedType() {
+      return new GenericTypeInfo<>(ByteBuffer.class);
+    }
+  }
+
+  public static class StripIdsMap<T> implements
+      FlatMapFunction<WindowedValue<ValueWithRecordId<T>>, WindowedValue<T>> {
+
+    @Override
+    public void flatMap(WindowedValue<ValueWithRecordId<T>> value,
+                        Collector<WindowedValue<T>> collector) throws Exception {
+      collector.collect(value.withValue(value.getValue().getValue()));
+    }
+
+  }
+
   private static class BoundedReadSourceTranslator<T>
       extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<Read.Bounded<T>> {
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/DedupingOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/DedupingOperator.java
new file mode 100644
index 000000000000..b8b40fe06169
--- /dev/null
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/DedupingOperator.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink.translation.wrappers.streaming.io;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.nio.ByteBuffer;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.beam.runners.flink.translation.wrappers.streaming.state.KeyGroupCheckpointedOperator;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder.Context;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.ValueWithRecordId;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider;
+import org.apache.flink.runtime.state.KeyGroupsList;
+import org.apache.flink.runtime.state.KeyedStateBackend;
+import org.apache.flink.runtime.state.KeyedStateCheckpointOutputStream;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.joda.time.Duration;
+
+/**
+ * Remove values with duplicate ids.
+ */
+public class DedupingOperator<T> extends AbstractStreamOperator<WindowedValue<T>>
+    implements OneInputStreamOperator<WindowedValue<ValueWithRecordId<T>>, WindowedValue<T>>,
+    KeyGroupCheckpointedOperator {
+
+  private static final long MAX_RETENTION_SINCE_ACCESS = Duration.standardMinutes(10L).getMillis();
+  private static final long MAX_CACHE_SIZE = 100_000L;
+
+  private transient LoadingCache<Integer, LoadingCache<ByteBuffer, AtomicBoolean>> dedupingCache;
+  private transient KeyedStateBackend<ByteBuffer> keyedStateBackend;
+
+  @Override
+  public void open() throws Exception {
+    super.open();
+    checkInitCache();
+    keyedStateBackend = getKeyedStateBackend();
+  }
+
+  private void checkInitCache() {
+    if (dedupingCache == null) {
+      dedupingCache = CacheBuilder.newBuilder().build(new KeyGroupLoader());
+    }
+  }
+
+  private static class KeyGroupLoader extends
+      CacheLoader<Integer, LoadingCache<ByteBuffer, AtomicBoolean>> {
+    @Override
+    public LoadingCache<ByteBuffer, AtomicBoolean> load(Integer ignore) throws Exception {
+      return CacheBuilder.newBuilder()
+          .expireAfterAccess(MAX_RETENTION_SINCE_ACCESS, TimeUnit.MILLISECONDS)
+          .maximumSize(MAX_CACHE_SIZE).build(new TrueBooleanLoader());
+    }
+  }
+
+  private static class TrueBooleanLoader extends CacheLoader<ByteBuffer, AtomicBoolean> {
+    @Override
+    public AtomicBoolean load(ByteBuffer ignore) throws Exception {
+      return new AtomicBoolean(true);
+    }
+  }
+
+  @Override
+  public void processElement(
+      StreamRecord<WindowedValue<ValueWithRecordId<T>>> streamRecord) throws Exception {
+    ByteBuffer currentKey = keyedStateBackend.getCurrentKey();
+    int groupIndex = keyedStateBackend.getCurrentKeyGroupIndex();
+    if (shouldOutput(groupIndex, currentKey)) {
+      WindowedValue<ValueWithRecordId<T>> value = streamRecord.getValue();
+      output.collect(streamRecord.replace(value.withValue(value.getValue().getValue())));
+    }
+  }
+
+  private boolean shouldOutput(int groupIndex, ByteBuffer id) throws ExecutionException {
+    return dedupingCache.get(groupIndex).getUnchecked(id).getAndSet(false);
+  }
+
+  @Override
+  public void restoreKeyGroupState(int keyGroupIndex, DataInputStream in) throws Exception {
+    checkInitCache();
+    Integer size = VarIntCoder.of().decode(in, Context.NESTED);
+    for (int i = 0; i < size; i++) {
+      byte[] idBytes = ByteArrayCoder.of().decode(in, Context.NESTED);
+      // restore the ids which not expired.
+      shouldOutput(keyGroupIndex, ByteBuffer.wrap(idBytes));
+    }
+  }
+
+  @Override
+  public void snapshotKeyGroupState(int keyGroupIndex, DataOutputStream out) throws Exception {
+    Set<ByteBuffer> ids = dedupingCache.get(keyGroupIndex).asMap().keySet();
+    VarIntCoder.of().encode(ids.size(), out, Context.NESTED);
+    for (ByteBuffer id : ids) {
+      ByteArrayCoder.of().encode(id.array(), out, Context.NESTED);
+    }
+  }
+
+  @Override
+  public void snapshotState(StateSnapshotContext context) throws Exception {
+    // copy from AbstractStreamOperator
+    if (getKeyedStateBackend() != null) {
+      KeyedStateCheckpointOutputStream out;
+
+      try {
+        out = context.getRawKeyedOperatorStateOutput();
+      } catch (Exception exception) {
+        throw new Exception("Could not open raw keyed operator state stream for "
+            + getOperatorName() + '.', exception);
+      }
+
+      try {
+        KeyGroupsList allKeyGroups = out.getKeyGroupList();
+        for (int keyGroupIdx : allKeyGroups) {
+          out.startNewKeyGroup(keyGroupIdx);
+
+          DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out);
+
+          // if (this instanceof KeyGroupCheckpointedOperator)
+          snapshotKeyGroupState(keyGroupIdx, dov);
+
+        }
+      } catch (Exception exception) {
+        throw new Exception("Could not write timer service of " + getOperatorName()
+            + " to checkpoint state stream.", exception);
+      } finally {
+        try {
+          out.close();
+        } catch (Exception closeException) {
+          LOG.warn("Could not close raw keyed operator state stream for {}. This "
+                  + "might have prevented deleting some state data.", getOperatorName(),
+              closeException);
+        }
+      }
+    }
+  }
+
+  @Override
+  public void initializeState(StateInitializationContext context) throws Exception {
+    if (getKeyedStateBackend() != null) {
+      KeyGroupsList localKeyGroupRange = getKeyedStateBackend().getKeyGroupRange();
+
+      for (KeyGroupStatePartitionStreamProvider streamProvider : context.getRawKeyedStateInputs()) {
+        DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(streamProvider.getStream());
+
+        int keyGroupIdx = streamProvider.getKeyGroupId();
+        checkArgument(localKeyGroupRange.contains(keyGroupIdx),
+            "Key Group " + keyGroupIdx + " does not belong to the local range.");
+
+        // if (this instanceof KeyGroupRestoringOperator)
+        restoreKeyGroupState(keyGroupIdx, div);
+
+      }
+    }
+  }
+
+}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
index b9c431d89377..ec2169950f6f 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
@@ -36,6 +36,7 @@
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.ValueWithRecordId;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.StoppableFunction;
 import org.apache.flink.api.common.state.ListState;
@@ -60,7 +61,7 @@
  */
 public class UnboundedSourceWrapper<
     OutputT, CheckpointMarkT extends UnboundedSource.CheckpointMark>
-    extends RichParallelSourceFunction<WindowedValue<OutputT>>
+    extends RichParallelSourceFunction<WindowedValue<ValueWithRecordId<OutputT>>>
     implements ProcessingTimeCallback, StoppableFunction,
     CheckpointListener, CheckpointedFunction {
 
@@ -113,7 +114,7 @@ public class UnboundedSourceWrapper<
    * Make it a field so that we can access it in {@link #onProcessingTime(long)} for emitting
    * watermarks.
    */
-  private transient SourceContext<WindowedValue<OutputT>> context;
+  private transient SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> context;
 
   /**
    * Pending checkpoints which have not been acknowledged yet.
@@ -210,7 +211,7 @@ public void open(Configuration parameters) throws Exception {
   }
 
   @Override
-  public void run(SourceContext<WindowedValue<OutputT>> ctx) throws Exception {
+  public void run(SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> ctx) throws Exception {
 
     context = ctx;
 
@@ -309,17 +310,19 @@ public void run(SourceContext<WindowedValue<OutputT>> ctx) throws Exception {
    * Emit the current element from the given Reader. The reader is guaranteed to have data.
    */
   private void emitElement(
-      SourceContext<WindowedValue<OutputT>> ctx,
+      SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> ctx,
       UnboundedSource.UnboundedReader<OutputT> reader) {
     // make sure that reader state update and element emission are atomic
     // with respect to snapshots
     synchronized (ctx.getCheckpointLock()) {
 
       OutputT item = reader.getCurrent();
+      byte[] recordId = reader.getCurrentRecordId();
       Instant timestamp = reader.getCurrentTimestamp();
 
-      WindowedValue<OutputT> windowedValue =
-          WindowedValue.of(item, timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
+      WindowedValue<ValueWithRecordId<OutputT>> windowedValue =
+          WindowedValue.of(new ValueWithRecordId<>(item, recordId), timestamp,
+              GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
       ctx.collectWithTimestamp(windowedValue, timestamp.getMillis());
     }
   }
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DedupingOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DedupingOperatorTest.java
new file mode 100644
index 000000000000..81efa34bf900
--- /dev/null
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/DedupingOperatorTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink.streaming;
+
+import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import java.nio.ByteBuffer;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.flink.translation.wrappers.streaming.io.DedupingOperator;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.ValueWithRecordId;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link DedupingOperator}.
+ */
+@RunWith(JUnit4.class)
+public class DedupingOperatorTest {
+
+  @Test
+  public void testDeduping() throws Exception {
+
+    KeyedOneInputStreamOperatorTestHarness<
+        ByteBuffer,
+        WindowedValue<ValueWithRecordId<String>>,
+        WindowedValue<String>> harness = getDebupingHarness();
+
+    harness.open();
+
+    String key1 = "key1";
+    String key2 = "key2";
+
+    harness.processElement(new StreamRecord<>(
+        WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key1, key1.getBytes()))));
+
+    harness.processElement(new StreamRecord<>(
+        WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key2, key2.getBytes()))));
+
+    harness.processElement(new StreamRecord<>(
+        WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key1, key1.getBytes()))));
+
+    assertThat(
+        this.<String>stripStreamRecordFromWindowedValue(harness.getOutput()),
+        contains(WindowedValue.valueInGlobalWindow(key1),
+            WindowedValue.valueInGlobalWindow(key2)));
+
+    OperatorStateHandles snapshot = harness.snapshot(0L, 0L);
+
+    harness.close();
+
+    harness = getDebupingHarness();
+    harness.setup();
+    harness.initializeState(snapshot);
+    harness.open();
+
+    String key3 = "key3";
+
+    harness.processElement(new StreamRecord<>(
+        WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key2, key2.getBytes()))));
+
+    harness.processElement(new StreamRecord<>(
+        WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key3, key3.getBytes()))));
+
+    assertThat(
+        this.<String>stripStreamRecordFromWindowedValue(harness.getOutput()),
+        contains(WindowedValue.valueInGlobalWindow(key3)));
+
+    harness.close();
+  }
+
+  private KeyedOneInputStreamOperatorTestHarness<ByteBuffer,
+      WindowedValue<ValueWithRecordId<String>>,
+      WindowedValue<String>> getDebupingHarness() throws Exception {
+    DedupingOperator<String> operator = new DedupingOperator<>();
+
+    return new KeyedOneInputStreamOperatorTestHarness<>(operator,
+        new KeySelector<WindowedValue<ValueWithRecordId<String>>, ByteBuffer>() {
+      @Override
+      public ByteBuffer getKey(WindowedValue<ValueWithRecordId<String>> value) throws Exception {
+        return ByteBuffer.wrap(value.getValue().getId());
+      }
+    }, TypeInformation.of(ByteBuffer.class));
+  }
+
+  private <T> Iterable<WindowedValue<T>> stripStreamRecordFromWindowedValue(
+      Iterable<Object> input) {
+
+    return FluentIterable.from(input).filter(new Predicate<Object>() {
+      @Override
+      public boolean apply(@Nullable Object o) {
+        return o instanceof StreamRecord && ((StreamRecord) o).getValue() instanceof WindowedValue;
+      }
+    }).transform(new Function<Object, WindowedValue<T>>() {
+      @Nullable
+      @Override
+      @SuppressWarnings({"unchecked", "rawtypes"})
+      public WindowedValue<T> apply(@Nullable Object o) {
+        if (o instanceof StreamRecord && ((StreamRecord) o).getValue() instanceof WindowedValue) {
+          return (WindowedValue) ((StreamRecord) o).getValue();
+        }
+        throw new RuntimeException("unreachable");
+      }
+    });
+  }
+}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
index 0cb528a8141e..500fa664bfa8 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
@@ -37,6 +37,7 @@
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.ValueWithRecordId;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.state.ListState;
@@ -116,7 +117,7 @@ public void testReaders() throws Exception {
       assertEquals(numSplits, flinkWrapper.getSplitSources().size());
 
       StreamSource<WindowedValue<
-          KV<Integer, Integer>>,
+          ValueWithRecordId<KV<Integer, Integer>>>,
           UnboundedSourceWrapper<
               KV<Integer, Integer>,
               TestCountingSource.CounterMark>> sourceOperator = new StreamSource<>(flinkWrapper);
@@ -126,7 +127,7 @@ public void testReaders() throws Exception {
       try {
         sourceOperator.open();
         sourceOperator.run(checkpointLock,
-            new Output<StreamRecord<WindowedValue<KV<Integer, Integer>>>>() {
+            new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
               private int count = 0;
 
               @Override
@@ -138,8 +139,8 @@ public void emitLatencyMarker(LatencyMarker latencyMarker) {
               }
 
               @Override
-              public void collect(
-                  StreamRecord<WindowedValue<KV<Integer, Integer>>> windowedValueStreamRecord) {
+              public void collect(StreamRecord<WindowedValue<
+                  ValueWithRecordId<KV<Integer, Integer>>>> windowedValueStreamRecord) {
 
                 count++;
                 if (count >= numElements) {
@@ -184,7 +185,7 @@ public void testRestore() throws Exception {
       assertEquals(numSplits, flinkWrapper.getSplitSources().size());
 
       StreamSource<
-          WindowedValue<KV<Integer, Integer>>,
+          WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
           UnboundedSourceWrapper<
               KV<Integer, Integer>,
               TestCountingSource.CounterMark>> sourceOperator = new StreamSource<>(flinkWrapper);
@@ -214,7 +215,7 @@ public void testRestore() throws Exception {
       try {
         sourceOperator.open();
         sourceOperator.run(checkpointLock,
-            new Output<StreamRecord<WindowedValue<KV<Integer, Integer>>>>() {
+            new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
               private int count = 0;
 
               @Override
@@ -226,10 +227,10 @@ public void emitLatencyMarker(LatencyMarker latencyMarker) {
               }
 
               @Override
-              public void collect(
-                  StreamRecord<WindowedValue<KV<Integer, Integer>>> windowedValueStreamRecord) {
+              public void collect(StreamRecord<WindowedValue<
+                  ValueWithRecordId<KV<Integer, Integer>>>> windowedValueStreamRecord) {
 
-                emittedElements.add(windowedValueStreamRecord.getValue().getValue());
+                emittedElements.add(windowedValueStreamRecord.getValue().getValue().getValue());
                 count++;
                 if (count >= numElements / 2) {
                   throw new SuccessException();
@@ -275,7 +276,7 @@ public void close() {
       assertEquals(numSplits, restoredFlinkWrapper.getSplitSources().size());
 
       StreamSource<
-          WindowedValue<KV<Integer, Integer>>,
+          WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
           UnboundedSourceWrapper<
               KV<Integer, Integer>,
               TestCountingSource.CounterMark>> restoredSourceOperator =
@@ -292,7 +293,7 @@ public void close() {
       try {
         restoredSourceOperator.open();
         restoredSourceOperator.run(checkpointLock,
-            new Output<StreamRecord<WindowedValue<KV<Integer, Integer>>>>() {
+            new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
               private int count = 0;
 
               @Override
@@ -304,9 +305,9 @@ public void emitLatencyMarker(LatencyMarker latencyMarker) {
               }
 
               @Override
-              public void collect(
-                  StreamRecord<WindowedValue<KV<Integer, Integer>>> windowedValueStreamRecord) {
-                emittedElements.add(windowedValueStreamRecord.getValue().getValue());
+              public void collect(StreamRecord<WindowedValue<
+                  ValueWithRecordId<KV<Integer, Integer>>>> windowedValueStreamRecord) {
+                emittedElements.add(windowedValueStreamRecord.getValue().getValue().getValue());
                 count++;
                 if (count >= numElements / 2) {
                   throw new SuccessException();

From 44c9385ddbcfbdf904c5d054754dca3ddde88ec2 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 17:16:34 -0700
Subject: [PATCH 111/387] [BEAM-2190] pom.xml: do a better job of dependency
 management

Even if Beam appears to have the correct dependencies, we cannot
guarantee that modules that depend on us transitively get the right
dependencies. For example, even though grpc-protobuf-lite has
protobuf-lite excluded, and the Maven Enforcer banned-dependencies
check passes... if a user happens to get a transitive dependency on
grpc-all first, they may pull in grpc-protobuf from that other source
without the exclusion. Thus we need to exclude protobuf-lite from
grpc-all as well.

While we're here, also add guava-jdk5 to the set of banned dependencies,
though (as above) we cannot currently properly identify the places it
might be transitively exposed in a users' pom.xml.
---
 pom.xml                                    | 24 ++++++++++++++++++++++
 sdks/java/io/google-cloud-platform/pom.xml |  8 +-------
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/pom.xml b/pom.xml
index 367a61f2f966..e3fa7d825381 100644
--- a/pom.xml
+++ b/pom.xml
@@ -553,6 +553,12 @@
         <groupId>io.grpc</groupId>
         <artifactId>grpc-all</artifactId>
         <version>${grpc.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-lite</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
 
       <dependency>
@@ -898,6 +904,12 @@
         <groupId>com.google.api.grpc</groupId>
         <artifactId>grpc-google-common-protos</artifactId>
         <version>${grpc-google-common-protos.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-lite</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
 
       <dependency>
@@ -1611,12 +1623,24 @@
                   <!-- Keep aligned with preqrequisite section below. -->
                   <version>[3.2,)</version>
                 </requireMavenVersion>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>enforce-banned-dependencies</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
                 <bannedDependencies>
                   <excludes>
+                    <exclude>com.google.guava:guava-jdk5</exclude>
                     <exclude>com.google.protobuf:protobuf-lite</exclude>
                   </excludes>
                 </bannedDependencies>
               </rules>
+              <fail>true</fail>
             </configuration>
           </execution>
         </executions>
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 6846990c98d0..f0526f01abb0 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -133,19 +133,13 @@
     </dependency>
 
     <!-- grpc-all does not obey IWYU, so we need to exclude from compile
-     scope and depend on it at runtime. -->
+         scope and depend on it at runtime. -->
     <dependency>
       <groupId>io.grpc</groupId>
       <artifactId>grpc-all</artifactId>
       <scope>runtime</scope>
     </dependency>
 
-    <dependency>
-      <groupId>io.grpc</groupId>
-      <artifactId>grpc-protobuf</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>

From 5bac40efcc7bcd9d53183579ec6089c23aeda782 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 19:06:03 -0700
Subject: [PATCH 112/387] [BEAM-2212] FileBasedSource: improve message when
 logging.

ValueProvider should not be printed, rather the string instead.
---
 .../src/main/java/org/apache/beam/sdk/io/FileBasedSource.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
index 2beb5e028069..4e0734265262 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
@@ -268,7 +268,7 @@ public final List<? extends FileBasedSource<T>> split(
       LOG.info(
           "Splitting filepattern {} into bundles of size {} took {} ms "
               + "and produced {} files and {} bundles",
-          fileOrPatternSpec,
+          fileOrPatternSpec.get(),
           desiredBundleSizeBytes,
           System.currentTimeMillis() - startTime,
           expandedFiles.size(),
@@ -327,7 +327,7 @@ public final BoundedReader<T> createReader(PipelineOptions options) throws IOExc
       }
       LOG.debug(
           "Creating a reader for file pattern {} took {} ms",
-          fileOrPatternSpec,
+          fileOrPatternSpec.get(),
           System.currentTimeMillis() - startTime);
       if (fileReaders.size() == 1) {
         return fileReaders.get(0);

From 9423babd8f827e843723c218441e9a91aaa7b361 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Mon, 8 May 2017 09:59:16 -0700
Subject: [PATCH 113/387] [BEAM-2212] FileBasedSource: refactor to remove uses
 of fileOrPatternSpec.get()

Makes it less likely to have errors from printing ValueProviders instead of runtime values
---
 .../apache/beam/sdk/io/FileBasedSource.java   | 39 ++++++++++++-------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
index 4e0734265262..d4413c90765e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java
@@ -196,19 +196,20 @@ public final long getEstimatedSizeBytes(PipelineOptions options) throws IOExcept
     // This implementation of method getEstimatedSizeBytes is provided to simplify subclasses. Here
     // we perform the size estimation of files and file patterns using the interface provided by
     // FileSystem.
+    checkState(
+        fileOrPatternSpec.isAccessible(),
+        "Cannot estimate size of a FileBasedSource with inaccessible file pattern: {}.",
+        fileOrPatternSpec);
+    String fileOrPattern = fileOrPatternSpec.get();
 
     if (mode == Mode.FILEPATTERN) {
-      checkState(fileOrPatternSpec.isAccessible(),
-                 "Size estimation should be done at execution time.");
-      String pattern = fileOrPatternSpec.get();
       long totalSize = 0;
-      List<MatchResult> inputs =
-          FileSystems.match(Collections.singletonList(pattern));
+      List<MatchResult> inputs = FileSystems.match(Collections.singletonList(fileOrPattern));
       MatchResult result = Iterables.getOnlyElement(inputs);
       checkArgument(
           result.status() == Status.OK,
           "Error matching the pattern or glob %s: status %s",
-          pattern,
+          fileOrPattern,
           result.status());
       List<Metadata> allMatches = result.metadata();
       for (Metadata metadata : allMatches) {
@@ -216,7 +217,7 @@ public final long getEstimatedSizeBytes(PipelineOptions options) throws IOExcept
       }
       LOG.info(
           "Filepattern {} matched {} files with total size {}",
-          fileOrPatternSpec.get(),
+          fileOrPattern,
           allMatches.size(),
           totalSize);
       return totalSize;
@@ -245,14 +246,17 @@ public final List<? extends FileBasedSource<T>> split(
     // split a FileBasedSource based on a file pattern to FileBasedSources based on full single
     // files. For files that can be efficiently seeked, we further split FileBasedSources based on
     // those files to FileBasedSources based on sub ranges of single files.
+    checkState(
+        fileOrPatternSpec.isAccessible(),
+        "Cannot split a FileBasedSource without access to the file or pattern specification: {}.",
+        fileOrPatternSpec);
+    String fileOrPattern = fileOrPatternSpec.get();
 
     if (mode == Mode.FILEPATTERN) {
       long startTime = System.currentTimeMillis();
-      checkState(fileOrPatternSpec.isAccessible(),
-                 "Bundle splitting should only happen at execution time.");
-      List<Metadata> expandedFiles = FileBasedSource.expandFilePattern(fileOrPatternSpec.get());
+      List<Metadata> expandedFiles = FileBasedSource.expandFilePattern(fileOrPattern);
       checkArgument(!expandedFiles.isEmpty(),
-          "Unable to find any files matching %s", fileOrPatternSpec.get());
+          "Unable to find any files matching %s", fileOrPattern);
       List<FileBasedSource<T>> splitResults = new ArrayList<>(expandedFiles.size());
       for (Metadata metadata : expandedFiles) {
         FileBasedSource<T> split = createForSubrangeOfFile(metadata, 0, metadata.sizeBytes());
@@ -268,7 +272,7 @@ public final List<? extends FileBasedSource<T>> split(
       LOG.info(
           "Splitting filepattern {} into bundles of size {} took {} ms "
               + "and produced {} files and {} bundles",
-          fileOrPatternSpec.get(),
+          fileOrPattern,
           desiredBundleSizeBytes,
           System.currentTimeMillis() - startTime,
           expandedFiles.size(),
@@ -283,7 +287,7 @@ public final List<? extends FileBasedSource<T>> split(
       } else {
         LOG.debug("The source for file {} is not split into sub-range based sources since "
             + "the file is not seekable",
-            fileOrPatternSpec);
+            fileOrPattern);
         return ImmutableList.of(this);
       }
     }
@@ -315,10 +319,15 @@ protected boolean isSplittable() throws Exception {
   public final BoundedReader<T> createReader(PipelineOptions options) throws IOException {
     // Validate the current source prior to creating a reader for it.
     this.validate();
+    checkState(
+        fileOrPatternSpec.isAccessible(),
+        "Cannot create a file reader without access to the file or pattern specification: {}.",
+        fileOrPatternSpec);
+    String fileOrPattern = fileOrPatternSpec.get();
 
     if (mode == Mode.FILEPATTERN) {
       long startTime = System.currentTimeMillis();
-      List<Metadata> fileMetadata = FileBasedSource.expandFilePattern(fileOrPatternSpec.get());
+      List<Metadata> fileMetadata = FileBasedSource.expandFilePattern(fileOrPattern);
       List<FileBasedReader<T>> fileReaders = new ArrayList<>();
       for (Metadata metadata : fileMetadata) {
         long endOffset = metadata.sizeBytes();
@@ -327,7 +336,7 @@ public final BoundedReader<T> createReader(PipelineOptions options) throws IOExc
       }
       LOG.debug(
           "Creating a reader for file pattern {} took {} ms",
-          fileOrPatternSpec.get(),
+          fileOrPattern,
           System.currentTimeMillis() - startTime);
       if (fileReaders.size() == 1) {
         return fileReaders.get(0);

From 13e51c9cd114eec73c47b71f46214dd92ee81048 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 5 May 2017 19:49:29 -0700
Subject: [PATCH 114/387] Convert Executor Services to use Daemon Threads

This will cause the DirectRunner to automatically shut down when the
worker threads are shut down.
---
 .../runners/direct/BoundedReadEvaluatorFactory.java | 13 ++++++++++++-
 .../apache/beam/runners/direct/DirectMetrics.java   | 10 +++++++++-
 .../beam/runners/direct/EvaluationContext.java      |  3 +--
 .../direct/ExecutorServiceParallelExecutor.java     | 12 +++++++++++-
 .../SplittableProcessElementsEvaluatorFactory.java  | 11 ++++++++++-
 5 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
index 99a0fca2e72d..76db861897e0 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
@@ -21,7 +21,9 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.SettableFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
@@ -57,7 +59,16 @@ final class BoundedReadEvaluatorFactory implements TransformEvaluatorFactory {
    */
   private static final long REQUIRED_DYNAMIC_SPLIT_ORIGINAL_SIZE = 0;
   private final EvaluationContext evaluationContext;
-  @VisibleForTesting final ExecutorService executor = Executors.newCachedThreadPool();
+
+  // TODO: (BEAM-723) Create a shared ExecutorService for maintenance tasks in the DirectRunner.
+  @VisibleForTesting
+  final ExecutorService executor =
+      Executors.newCachedThreadPool(
+          new ThreadFactoryBuilder()
+              .setThreadFactory(MoreExecutors.platformThreadFactory())
+              .setDaemon(true)
+              .setNameFormat("direct-dynamic-split-requester")
+              .build());
 
   private final long minimumDynamicSplitSize;
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
index b6ca492e6f25..b7cd6e73f07a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
@@ -21,6 +21,8 @@
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -51,7 +53,13 @@
 class DirectMetrics extends MetricResults {
 
   // TODO: (BEAM-723) Create a shared ExecutorService for maintenance tasks in the DirectRunner.
-  private static final ExecutorService COUNTER_COMMITTER = Executors.newCachedThreadPool();
+  private static final ExecutorService COUNTER_COMMITTER =
+      Executors.newCachedThreadPool(
+          new ThreadFactoryBuilder()
+              .setThreadFactory(MoreExecutors.platformThreadFactory())
+              .setDaemon(true)
+              .setNameFormat("direct-metrics-counter-committer")
+              .build());
 
   private interface MetricAggregation<UpdateT, ResultT> {
     UpdateT zero();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
index 362ff91b893c..c62711996648 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -123,8 +123,7 @@ private EvaluationContext(
     this.applicationStateInternals = new ConcurrentHashMap<>();
     this.metrics = new DirectMetrics();
 
-    this.callbackExecutor =
-        WatermarkCallbackExecutor.create(MoreExecutors.directExecutor());
+    this.callbackExecutor = WatermarkCallbackExecutor.create(MoreExecutors.directExecutor());
   }
 
   public void initialize(
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index 02fb11aea315..b7f473241e9a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -28,6 +28,8 @@
 import com.google.common.cache.RemovalListener;
 import com.google.common.cache.RemovalNotification;
 import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -132,7 +134,15 @@ private ExecutorServiceParallelExecutor(
       Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>> transformEnforcements,
       EvaluationContext context) {
     this.targetParallelism = targetParallelism;
-    this.executorService = Executors.newFixedThreadPool(targetParallelism);
+    // Don't use Daemon threads for workers. The Pipeline should continue to execute even if there
+    // are no other active threads (for example, because waitUntilFinish was not called)
+    this.executorService =
+        Executors.newFixedThreadPool(
+            targetParallelism,
+            new ThreadFactoryBuilder()
+                .setThreadFactory(MoreExecutors.platformThreadFactory())
+                .setNameFormat("direct-runner-worker")
+                .build());
     this.graph = graph;
     this.rootProviderRegistry = rootProviderRegistry;
     this.registry = registry;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index 27972335d703..f490b0ba49d0 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -17,6 +17,8 @@
  */
 package org.apache.beam.runners.direct;
 
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.Executors;
@@ -171,7 +173,14 @@ public <AdditionalOutputT> void outputWindowedValue(
             outputWindowedValue,
             evaluationContext.createSideInputReader(transform.getSideInputs()),
             // TODO: For better performance, use a higher-level executor?
-            Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory()),
+            // TODO: (BEAM-723) Create a shared ExecutorService for maintenance tasks in the
+            // DirectRunner.
+            Executors.newSingleThreadScheduledExecutor(
+                new ThreadFactoryBuilder()
+                    .setThreadFactory(MoreExecutors.platformThreadFactory())
+                    .setDaemon(true)
+                    .setNameFormat("direct-splittable-process-element-checkpoint-executor")
+                    .build()),
             10000,
             Duration.standardSeconds(10)));
 

From 50a4c56de1d3aa0e51e8b2b08b71715be4c2ef80 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Sun, 7 May 2017 13:55:20 -0700
Subject: [PATCH 115/387] [BEAM-2206] Move pipelineOptions into options modules

---
 .../examples/complete/autocomplete.py         |   4 +-
 .../examples/complete/estimate_pi.py          |   4 +-
 .../complete/game/hourly_team_score.py        |   6 +-
 .../examples/complete/game/user_score.py      |   4 +-
 .../apache_beam/examples/complete/tfidf.py    |   4 +-
 .../complete/top_wikipedia_sessions.py        |   4 +-
 .../examples/cookbook/bigquery_side_input.py  |   4 +-
 .../apache_beam/examples/cookbook/coders.py   |   4 +-
 .../examples/cookbook/custom_ptransform.py    |   2 +-
 .../examples/cookbook/datastore_wordcount.py  |   6 +-
 .../examples/cookbook/group_with_coder.py     |   4 +-
 .../examples/cookbook/mergecontacts.py        |   4 +-
 .../cookbook/multiple_output_pardo.py         |   4 +-
 .../apache_beam/examples/snippets/snippets.py |  34 +-
 .../examples/snippets/snippets_test.py        |   2 +-
 sdks/python/apache_beam/examples/wordcount.py |   4 +-
 .../examples/wordcount_debugging.py           |   4 +-
 .../apache_beam/examples/wordcount_minimal.py |   4 +-
 .../apache_beam/internal/gcp/json_value.py    |   2 +-
 .../internal/gcp/json_value_test.py           |   4 +-
 sdks/python/apache_beam/io/filebasedsource.py |   6 +-
 .../apache_beam/io/filebasedsource_test.py    |   4 +-
 sdks/python/apache_beam/io/fileio.py          |   6 +-
 sdks/python/apache_beam/io/fileio_test.py     |   2 +-
 sdks/python/apache_beam/io/gcp/bigquery.py    |   2 +-
 .../apache_beam/io/gcp/bigquery_test.py       |   2 +-
 sdks/python/apache_beam/options/__init__.py   |  16 +
 .../apache_beam/options/pipeline_options.py   | 627 ++++++++++++++++++
 .../pipeline_options_test.py                  |   6 +-
 .../pipeline_options_validator.py             |  14 +-
 .../pipeline_options_validator_test.py        |   4 +-
 .../apache_beam/options/value_provider.py     | 103 +++
 .../{utils => options}/value_provider_test.py |   6 +-
 sdks/python/apache_beam/pipeline.py           |  10 +-
 .../runners/dataflow/dataflow_runner.py       |   2 +-
 .../runners/dataflow/dataflow_runner_test.py  |   2 +-
 .../runners/dataflow/internal/apiclient.py    |   8 +-
 .../dataflow/internal/apiclient_test.py       |   2 +-
 .../runners/dataflow/internal/dependency.py   |   4 +-
 .../dataflow/internal/dependency_test.py      |   6 +-
 .../runners/dataflow/template_runner_test.py  |   2 +-
 .../runners/dataflow/test_dataflow_runner.py  |   2 +-
 .../runners/direct/direct_runner.py           |   4 +-
 .../runners/direct/transform_evaluator.py     |   2 +-
 .../portability/maptask_executor_runner.py    |   2 +-
 .../python/apache_beam/runners/runner_test.py |   2 +-
 sdks/python/apache_beam/test_pipeline.py      |   2 +-
 sdks/python/apache_beam/test_pipeline_test.py |   2 +-
 sdks/python/apache_beam/transforms/core.py    |   2 +-
 sdks/python/apache_beam/transforms/display.py |   2 +-
 .../apache_beam/transforms/display_test.py    |   2 +-
 .../apache_beam/transforms/ptransform.py      |   2 +-
 .../apache_beam/transforms/ptransform_test.py |   2 +-
 .../typehints/typed_pipeline_test.py          |   2 +-
 .../apache_beam/utils/pipeline_options.py     | 612 +----------------
 .../apache_beam/utils/value_provider.py       |  88 +--
 56 files changed, 864 insertions(+), 806 deletions(-)
 create mode 100644 sdks/python/apache_beam/options/__init__.py
 create mode 100644 sdks/python/apache_beam/options/pipeline_options.py
 rename sdks/python/apache_beam/{utils => options}/pipeline_options_test.py (97%)
 rename sdks/python/apache_beam/{utils => options}/pipeline_options_validator.py (94%)
 rename sdks/python/apache_beam/{utils => options}/pipeline_options_validator_test.py (98%)
 create mode 100644 sdks/python/apache_beam/options/value_provider.py
 rename sdks/python/apache_beam/{utils => options}/value_provider_test.py (96%)

diff --git a/sdks/python/apache_beam/examples/complete/autocomplete.py b/sdks/python/apache_beam/examples/complete/autocomplete.py
index f954ec1a4365..f0acc3fc562d 100644
--- a/sdks/python/apache_beam/examples/complete/autocomplete.py
+++ b/sdks/python/apache_beam/examples/complete/autocomplete.py
@@ -26,8 +26,8 @@
 import apache_beam as beam
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 def run(argv=None):
diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py
index 3e57f717e2df..c709713bd97e 100644
--- a/sdks/python/apache_beam/examples/complete/estimate_pi.py
+++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py
@@ -37,8 +37,8 @@
 from apache_beam.typehints import Any
 from apache_beam.typehints import Iterable
 from apache_beam.typehints import Tuple
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 @beam.typehints.with_output_types(Tuple[int, int, int])
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
index 1d1cee8761da..e9d71881f8c9 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
@@ -59,9 +59,9 @@
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.typehints import with_input_types
 from apache_beam.typehints import with_output_types
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class ParseEventFn(beam.DoFn):
diff --git a/sdks/python/apache_beam/examples/complete/game/user_score.py b/sdks/python/apache_beam/examples/complete/game/user_score.py
index 02d47757ad0b..389d2c6a8433 100644
--- a/sdks/python/apache_beam/examples/complete/game/user_score.py
+++ b/sdks/python/apache_beam/examples/complete/game/user_score.py
@@ -49,8 +49,8 @@
 from apache_beam.metrics import Metrics
 from apache_beam.typehints import with_input_types
 from apache_beam.typehints import with_output_types
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 
 class ParseEventFn(beam.DoFn):
diff --git a/sdks/python/apache_beam/examples/complete/tfidf.py b/sdks/python/apache_beam/examples/complete/tfidf.py
index 367e27543983..a98d90640d52 100644
--- a/sdks/python/apache_beam/examples/complete/tfidf.py
+++ b/sdks/python/apache_beam/examples/complete/tfidf.py
@@ -32,8 +32,8 @@
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
 from apache_beam.pvalue import AsSingleton
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 def read_documents(pipeline, uris):
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
index c24eb7565c12..aa48e4eb920e 100644
--- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
+++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
@@ -52,8 +52,8 @@
 from apache_beam.transforms.window import FixedWindows
 from apache_beam.transforms.window import Sessions
 from apache_beam.transforms.window import TimestampedValue
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 ONE_HOUR_IN_SECONDS = 3600
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
index f68c95d03f4a..6b2881828a0d 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
@@ -36,8 +36,8 @@
 from apache_beam.io import WriteToText
 from apache_beam.pvalue import AsList
 from apache_beam.pvalue import AsSingleton
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 def create_groups(group_ids, corpus, word, ignore_corpus, ignore_word):
diff --git a/sdks/python/apache_beam/examples/cookbook/coders.py b/sdks/python/apache_beam/examples/cookbook/coders.py
index ede9d70c7bda..aeeb3c9128a5 100644
--- a/sdks/python/apache_beam/examples/cookbook/coders.py
+++ b/sdks/python/apache_beam/examples/cookbook/coders.py
@@ -37,8 +37,8 @@
 import apache_beam as beam
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class JsonCoder(object):
diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
index 67d1ff865bfe..609f2cd87181 100644
--- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
+++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
@@ -28,7 +28,7 @@
 import apache_beam as beam
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 
 # pylint doesn't understand our pipeline syntax:
diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
index 1f88398fbbad..411feb8058a7 100644
--- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
+++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
@@ -76,9 +76,9 @@
 from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore
 from apache_beam.metrics import Metrics
 from apache_beam.metrics.metric import MetricsFilter
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class WordExtractingDoFn(beam.DoFn):
diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
index cb675bdc6fac..6bdadae9f3d1 100644
--- a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
+++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
@@ -37,8 +37,8 @@
 from apache_beam.io import WriteToText
 from apache_beam.typehints import typehints
 from apache_beam.typehints.decorators import with_output_types
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class Player(object):
diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
index c880a9a0c2f0..5aaba1015aa9 100644
--- a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
+++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
@@ -38,8 +38,8 @@
 import apache_beam as beam
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 def run(argv=None, assert_results=None):
diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
index 9c82df4c46dd..9759f4821045 100644
--- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
+++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
@@ -59,8 +59,8 @@
 from apache_beam import pvalue
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class SplitLinesToWordsFn(beam.DoFn):
diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py
index 1b750b43e818..26af71da883e 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets.py
@@ -91,7 +91,7 @@ def filter_words(unused_x):
 
   # [START pipelines_constructing_creating]
   import apache_beam as beam
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   p = beam.Pipeline(options=PipelineOptions())
   # [END pipelines_constructing_creating]
@@ -126,7 +126,7 @@ def model_pipelines(argv):
   import re
 
   import apache_beam as beam
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   class MyOptions(PipelineOptions):
 
@@ -161,7 +161,7 @@ def _add_argparse_args(cls, parser):
 
 def model_pcollection(argv):
   """Creating a PCollection from data in local memory."""
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   class MyOptions(PipelineOptions):
 
@@ -197,7 +197,7 @@ def pipeline_options_remote(argv):
   """Creating a Pipeline using a PipelineOptions object for remote execution."""
 
   from apache_beam import Pipeline
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   # [START pipeline_options_create]
   options = PipelineOptions(flags=argv)
@@ -212,8 +212,8 @@ def _add_argparse_args(cls, parser):
       parser.add_argument('--output')
   # [END pipeline_options_define_custom]
 
-  from apache_beam.utils.pipeline_options import GoogleCloudOptions
-  from apache_beam.utils.pipeline_options import StandardOptions
+  from apache_beam.options.pipeline_options import GoogleCloudOptions
+  from apache_beam.options.pipeline_options import StandardOptions
 
   # [START pipeline_options_dataflow_service]
   # Create and set your PipelineOptions.
@@ -248,7 +248,7 @@ def pipeline_options_local(argv):
   """Creating a Pipeline using a PipelineOptions object for local execution."""
 
   from apache_beam import Pipeline
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   options = PipelineOptions(flags=argv)
 
@@ -341,7 +341,7 @@ def pipeline_monitoring(renames):
 
   import re
   import apache_beam as beam
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   class WordCountOptions(PipelineOptions):
 
@@ -405,9 +405,9 @@ def examples_wordcount_minimal(renames):
 
   import apache_beam as beam
 
-  from apache_beam.utils.pipeline_options import GoogleCloudOptions
-  from apache_beam.utils.pipeline_options import StandardOptions
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import GoogleCloudOptions
+  from apache_beam.options.pipeline_options import StandardOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   # [START examples_wordcount_minimal_options]
   options = PipelineOptions()
@@ -462,7 +462,7 @@ def examples_wordcount_wordcount(renames):
   import re
 
   import apache_beam as beam
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   argv = []
 
@@ -582,7 +582,7 @@ def process(self, element):
 from apache_beam.io import iobase
 from apache_beam.io.range_trackers import OffsetRangeTracker
 from apache_beam.transforms.core import PTransform
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 
 # Defining a new source.
@@ -747,7 +747,7 @@ def model_custom_sink(simplekv, KVs, final_table_name_no_ptransform,
   import apache_beam as beam
   from apache_beam.io import iobase
   from apache_beam.transforms.core import PTransform
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   # Defining the new sink.
   # [START model_custom_sink_new_sink]
@@ -841,7 +841,7 @@ def filter_words(x):
     return re.findall(r'[A-Za-z\']+', x)
 
   import apache_beam as beam
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   # [START model_textio_read]
   p = beam.Pipeline(options=PipelineOptions())
@@ -885,7 +885,7 @@ def model_datastoreio():
   from google.cloud.proto.datastore.v1 import query_pb2
   import googledatastore
   import apache_beam as beam
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
   from apache_beam.io.gcp.datastore.v1.datastoreio import ReadFromDatastore
   from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore
 
@@ -918,7 +918,7 @@ def to_entity(content):
 def model_bigqueryio():
   """Using a Read and Write transform to read/write to BigQuery."""
   import apache_beam as beam
-  from apache_beam.utils.pipeline_options import PipelineOptions
+  from apache_beam.options.pipeline_options import PipelineOptions
 
   # [START model_bigqueryio_read]
   p = beam.Pipeline(options=PipelineOptions())
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index 370b43644ece..014809628ee1 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -31,7 +31,7 @@
 from apache_beam import typehints
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.examples.snippets import snippets
 
 # pylint: disable=expression-not-assigned
diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py
index adcc4db1908e..8c6a485bc7a7 100644
--- a/sdks/python/apache_beam/examples/wordcount.py
+++ b/sdks/python/apache_beam/examples/wordcount.py
@@ -27,8 +27,8 @@
 from apache_beam.io import WriteToText
 from apache_beam.metrics import Metrics
 from apache_beam.metrics.metric import MetricsFilter
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class WordExtractingDoFn(beam.DoFn):
diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py
index 9779b82259b4..98acde4a4752 100644
--- a/sdks/python/apache_beam/examples/wordcount_debugging.py
+++ b/sdks/python/apache_beam/examples/wordcount_debugging.py
@@ -49,8 +49,8 @@
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
 from apache_beam.metrics import Metrics
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class FilterTextFn(beam.DoFn):
diff --git a/sdks/python/apache_beam/examples/wordcount_minimal.py b/sdks/python/apache_beam/examples/wordcount_minimal.py
index b80ed8465aba..5109c089f993 100644
--- a/sdks/python/apache_beam/examples/wordcount_minimal.py
+++ b/sdks/python/apache_beam/examples/wordcount_minimal.py
@@ -53,8 +53,8 @@
 import apache_beam as beam
 from apache_beam.io import ReadFromText
 from apache_beam.io import WriteToText
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 def run(argv=None):
diff --git a/sdks/python/apache_beam/internal/gcp/json_value.py b/sdks/python/apache_beam/internal/gcp/json_value.py
index 4099c1ad5eca..523db9c7d804 100644
--- a/sdks/python/apache_beam/internal/gcp/json_value.py
+++ b/sdks/python/apache_beam/internal/gcp/json_value.py
@@ -25,7 +25,7 @@
   extra_types = None
 # pylint: enable=wrong-import-order, wrong-import-position
 
-from apache_beam.utils.value_provider import ValueProvider
+from apache_beam.options.value_provider import ValueProvider
 
 
 _MAXINT64 = (1 << 63) - 1
diff --git a/sdks/python/apache_beam/internal/gcp/json_value_test.py b/sdks/python/apache_beam/internal/gcp/json_value_test.py
index a61c50bca899..b1fd63f048e9 100644
--- a/sdks/python/apache_beam/internal/gcp/json_value_test.py
+++ b/sdks/python/apache_beam/internal/gcp/json_value_test.py
@@ -21,8 +21,8 @@
 
 from apache_beam.internal.gcp.json_value import from_json_value
 from apache_beam.internal.gcp.json_value import to_json_value
-from apache_beam.utils.value_provider import StaticValueProvider
-from apache_beam.utils.value_provider import RuntimeValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import RuntimeValueProvider
 
 
 # Protect against environments where apitools library is not available.
diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py
index e25f92ebe9c4..215e01510224 100644
--- a/sdks/python/apache_beam/io/filebasedsource.py
+++ b/sdks/python/apache_beam/io/filebasedsource.py
@@ -32,9 +32,9 @@
 from apache_beam.io.filesystem import CompressionTypes
 from apache_beam.io.filesystems import FileSystems
 from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.utils.value_provider import ValueProvider
-from apache_beam.utils.value_provider import StaticValueProvider
-from apache_beam.utils.value_provider import check_accessible
+from apache_beam.options.value_provider import ValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import check_accessible
 
 MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 25
 
diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py
index f6cdf269828b..4ff23fc3d6a4 100644
--- a/sdks/python/apache_beam/io/filebasedsource_test.py
+++ b/sdks/python/apache_beam/io/filebasedsource_test.py
@@ -43,8 +43,8 @@
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
-from apache_beam.utils.value_provider import StaticValueProvider
-from apache_beam.utils.value_provider import RuntimeValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import RuntimeValueProvider
 
 
 class LineSource(FileBasedSource):
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index bfed96ea59e2..ca3a759797bb 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -31,9 +31,9 @@
 from apache_beam.io.filesystem import CompressionTypes
 from apache_beam.io.filesystems import FileSystems
 from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.utils.value_provider import ValueProvider
-from apache_beam.utils.value_provider import StaticValueProvider
-from apache_beam.utils.value_provider import check_accessible
+from apache_beam.options.value_provider import ValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import check_accessible
 
 DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN'
 
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 3f7211ede359..e0e9774dd255 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -35,7 +35,7 @@
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
 
-from apache_beam.utils.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
 
 
 # TODO: Refactor code so all io tests are using same library
diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py
index 4e8d61bb52c4..1ed28a82b668 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery.py
@@ -117,7 +117,7 @@
 from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
 from apache_beam.transforms.display import DisplayDataItem
 from apache_beam.utils import retry
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
 from apache_beam.io.gcp.internal.clients import bigquery
 
 # Protect against environments where bigquery library is not available.
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py
index 2b83079ba865..a26050c761c6 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py
@@ -34,7 +34,7 @@
 from apache_beam.internal.gcp.json_value import to_json_value
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 # Protect against environments where bigquery library is not available.
 # pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/options/__init__.py b/sdks/python/apache_beam/options/__init__.py
new file mode 100644
index 000000000000..cce3acad34a4
--- /dev/null
+++ b/sdks/python/apache_beam/options/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py
new file mode 100644
index 000000000000..b79d85db8595
--- /dev/null
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -0,0 +1,627 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Pipeline options obtained from command line parsing."""
+
+import argparse
+
+from apache_beam.transforms.display import HasDisplayData
+from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import RuntimeValueProvider
+from apache_beam.options.value_provider import ValueProvider
+
+
+def _static_value_provider_of(value_type):
+  """"Helper function to plug a ValueProvider into argparse.
+
+  Args:
+    value_type: the type of the value. Since the type param of argparse's
+                add_argument will always be ValueProvider, we need to
+                preserve the type of the actual value.
+  Returns:
+    A partially constructed StaticValueProvider in the form of a function.
+
+  """
+  def _f(value):
+    _f.func_name = value_type.__name__
+    return StaticValueProvider(value_type, value)
+  return _f
+
+
+class BeamArgumentParser(argparse.ArgumentParser):
+  """An ArgumentParser that supports ValueProvider options.
+
+  Example Usage::
+
+    class TemplateUserOptions(PipelineOptions):
+      @classmethod
+
+      def _add_argparse_args(cls, parser):
+        parser.add_value_provider_argument('--vp-arg1', default='start')
+        parser.add_value_provider_argument('--vp-arg2')
+        parser.add_argument('--non-vp-arg')
+
+  """
+  def add_value_provider_argument(self, *args, **kwargs):
+    """ValueProvider arguments can be either of type keyword or positional.
+    At runtime, even positional arguments will need to be supplied in the
+    key/value form.
+    """
+    # Extract the option name from positional argument ['pos_arg']
+    assert args != () and len(args[0]) >= 1
+    if args[0][0] != '-':
+      option_name = args[0]
+      if kwargs.get('nargs') is None:  # make them optionally templated
+        kwargs['nargs'] = '?'
+    else:
+      # or keyword arguments like [--kw_arg, -k, -w] or [--kw-arg]
+      option_name = [i.replace('--', '') for i in args if i[:2] == '--'][0]
+
+    # reassign the type to make room for using
+    # StaticValueProvider as the type for add_argument
+    value_type = kwargs.get('type') or str
+    kwargs['type'] = _static_value_provider_of(value_type)
+
+    # reassign default to default_value to make room for using
+    # RuntimeValueProvider as the default for add_argument
+    default_value = kwargs.get('default')
+    kwargs['default'] = RuntimeValueProvider(
+        option_name=option_name,
+        value_type=value_type,
+        default_value=default_value
+    )
+
+    # have add_argument do most of the work
+    self.add_argument(*args, **kwargs)
+
+
+class PipelineOptions(HasDisplayData):
+  """Pipeline options class used as container for command line options.
+
+  The class is essentially a wrapper over the standard argparse Python module
+  (see https://docs.python.org/3/library/argparse.html).  To define one option
+  or a group of options you subclass from PipelineOptions::
+
+    class XyzOptions(PipelineOptions):
+
+      @classmethod
+      def _add_argparse_args(cls, parser):
+        parser.add_argument('--abc', default='start')
+        parser.add_argument('--xyz', default='end')
+
+  The arguments for the add_argument() method are exactly the ones
+  described in the argparse public documentation.
+
+  Pipeline objects require an options object during initialization.
+  This is obtained simply by initializing an options class as defined above::
+
+    p = Pipeline(options=XyzOptions())
+    if p.options.xyz == 'end':
+      raise ValueError('Option xyz has an invalid value.')
+
+  By default the options classes will use command line arguments to initialize
+  the options.
+  """
+  def __init__(self, flags=None, **kwargs):
+    """Initialize an options class.
+
+    The initializer will traverse all subclasses, add all their argparse
+    arguments and then parse the command line specified by flags or by default
+    the one obtained from sys.argv.
+
+    The subclasses are not expected to require a redefinition of __init__.
+
+    Args:
+      flags: An iterable of command line arguments to be used. If not specified
+        then sys.argv will be used as input for parsing arguments.
+
+      **kwargs: Add overrides for arguments passed in flags.
+    """
+    self._flags = flags
+    self._all_options = kwargs
+    parser = BeamArgumentParser()
+
+    for cls in type(self).mro():
+      if cls == PipelineOptions:
+        break
+      elif '_add_argparse_args' in cls.__dict__:
+        cls._add_argparse_args(parser)
+    # The _visible_options attribute will contain only those options from the
+    # flags (i.e., command line) that can be recognized. The _all_options
+    # field contains additional overrides.
+    self._visible_options, _ = parser.parse_known_args(flags)
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    # Override this in subclasses to provide options.
+    pass
+
+  @classmethod
+  def from_dictionary(cls, options):
+    """Returns a PipelineOptions from a dictionary of arguments.
+
+    Args:
+      options: Dictinary of argument value pairs.
+
+    Returns:
+      A PipelineOptions object representing the given arguments.
+    """
+    flags = []
+    for k, v in options.iteritems():
+      if isinstance(v, bool):
+        if v:
+          flags.append('--%s' % k)
+      else:
+        flags.append('--%s=%s' % (k, v))
+
+    return cls(flags)
+
+  def get_all_options(self, drop_default=False):
+    """Returns a dictionary of all defined arguments.
+
+    Returns a dictionary of all defined arguments (arguments that are defined in
+    any subclass of PipelineOptions) into a dictionary.
+
+    Args:
+      drop_default: If set to true, options that are equal to their default
+        values, are not returned as part of the result dictionary.
+
+    Returns:
+      Dictionary of all args and values.
+    """
+
+    # TODO(BEAM-1319): PipelineOption sub-classes in the main session might be
+    # repeated. Pick last unique instance of each subclass to avoid conflicts.
+    subset = {}
+    parser = BeamArgumentParser()
+    for cls in PipelineOptions.__subclasses__():
+      subset[str(cls)] = cls
+    for cls in subset.values():
+      cls._add_argparse_args(parser)  # pylint: disable=protected-access
+    known_args, _ = parser.parse_known_args(self._flags)
+    result = vars(known_args)
+
+    # Apply the overrides if any
+    for k in result.keys():
+      if k in self._all_options:
+        result[k] = self._all_options[k]
+      if (drop_default and
+          parser.get_default(k) == result[k] and
+          not isinstance(parser.get_default(k), ValueProvider)):
+        del result[k]
+
+    return result
+
+  def display_data(self):
+    return self.get_all_options(True)
+
+  def view_as(self, cls):
+    view = cls(self._flags)
+    view._all_options = self._all_options
+    return view
+
+  def _visible_option_list(self):
+    return sorted(option
+                  for option in dir(self._visible_options) if option[0] != '_')
+
+  def __dir__(self):
+    return sorted(dir(type(self)) + self.__dict__.keys() +
+                  self._visible_option_list())
+
+  def __getattr__(self, name):
+    # Special methods which may be accessed before the object is
+    # fully constructed (e.g. in unpickling).
+    if name[:2] == name[-2:] == '__':
+      return object.__getattribute__(self, name)
+    elif name in self._visible_option_list():
+      return self._all_options.get(name, getattr(self._visible_options, name))
+    else:
+      raise AttributeError("'%s' object has no attribute '%s'" %
+                           (type(self).__name__, name))
+
+  def __setattr__(self, name, value):
+    if name in ('_flags', '_all_options', '_visible_options'):
+      super(PipelineOptions, self).__setattr__(name, value)
+    elif name in self._visible_option_list():
+      self._all_options[name] = value
+    else:
+      raise AttributeError("'%s' object has no attribute '%s'" %
+                           (type(self).__name__, name))
+
+  def __str__(self):
+    return '%s(%s)' % (type(self).__name__,
+                       ', '.join('%s=%s' % (option, getattr(self, option))
+                                 for option in self._visible_option_list()))
+
+
+class StandardOptions(PipelineOptions):
+
+  DEFAULT_RUNNER = 'DirectRunner'
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    parser.add_argument(
+        '--runner',
+        help=('Pipeline runner used to execute the workflow. Valid values are '
+              'DirectRunner, DataflowRunner.'))
+    # Whether to enable streaming mode.
+    parser.add_argument('--streaming',
+                        default=False,
+                        action='store_true',
+                        help='Whether to enable streaming mode.')
+
+  # TODO(BEAM-1265): Remove this error, once at least one runner supports
+  # streaming pipelines.
+  def validate(self, validator):
+    errors = []
+    if self.view_as(StandardOptions).streaming:
+      errors.append('Streaming pipelines are not supported.')
+    return errors
+
+
+class TypeOptions(PipelineOptions):
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    # TODO(laolu): Add a type inferencing option here once implemented.
+    parser.add_argument('--type_check_strictness',
+                        default='DEFAULT_TO_ANY',
+                        choices=['ALL_REQUIRED', 'DEFAULT_TO_ANY'],
+                        help='The level of exhaustive manual type-hint '
+                        'annotation required')
+    parser.add_argument('--no_pipeline_type_check',
+                        dest='pipeline_type_check',
+                        action='store_false',
+                        help='Disable type checking at pipeline construction '
+                        'time')
+    parser.add_argument('--runtime_type_check',
+                        default=False,
+                        action='store_true',
+                        help='Enable type checking at pipeline execution '
+                        'time. NOTE: only supported with the '
+                        'DirectRunner')
+
+
+class DirectOptions(PipelineOptions):
+  """DirectRunner-specific execution options."""
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    parser.add_argument(
+        '--no_direct_runner_use_stacked_bundle',
+        action='store_false',
+        dest='direct_runner_use_stacked_bundle',
+        help='DirectRunner uses stacked WindowedValues within a Bundle for '
+        'memory optimization. Set --no_direct_runner_use_stacked_bundle to '
+        'avoid it.')
+
+
+class GoogleCloudOptions(PipelineOptions):
+  """Google Cloud Dataflow service execution options."""
+
+  BIGQUERY_API_SERVICE = 'bigquery.googleapis.com'
+  COMPUTE_API_SERVICE = 'compute.googleapis.com'
+  STORAGE_API_SERVICE = 'storage.googleapis.com'
+  DATAFLOW_ENDPOINT = 'https://dataflow.googleapis.com'
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    parser.add_argument(
+        '--dataflow_endpoint',
+        default=cls.DATAFLOW_ENDPOINT,
+        help=
+        ('The URL for the Dataflow API. If not set, the default public URL '
+         'will be used.'))
+    # Remote execution must check that this option is not None.
+    parser.add_argument('--project',
+                        default=None,
+                        help='Name of the Cloud project owning the Dataflow '
+                        'job.')
+    # Remote execution must check that this option is not None.
+    parser.add_argument('--job_name',
+                        default=None,
+                        help='Name of the Cloud Dataflow job.')
+    # Remote execution must check that this option is not None.
+    parser.add_argument('--staging_location',
+                        default=None,
+                        help='GCS path for staging code packages needed by '
+                        'workers.')
+    # Remote execution must check that this option is not None.
+    # If staging_location is not set, it defaults to temp_location.
+    parser.add_argument('--temp_location',
+                        default=None,
+                        help='GCS path for saving temporary workflow jobs.')
+    # The Cloud Dataflow service does not yet honor this setting. However, once
+    # service support is added then users of this SDK will be able to control
+    # the region. Default is up to the Dataflow service. See
+    # https://cloud.google.com/compute/docs/regions-zones/regions-zones for a
+    # list of valid options/
+    parser.add_argument('--region',
+                        default='us-central1',
+                        help='The Google Compute Engine region for creating '
+                        'Dataflow job.')
+    parser.add_argument('--service_account_email',
+                        default=None,
+                        help='Identity to run virtual machines as.')
+    parser.add_argument('--no_auth', dest='no_auth', type=bool, default=False)
+    # Option to run templated pipelines
+    parser.add_argument('--template_location',
+                        default=None,
+                        help='Save job to specified local or GCS location.')
+
+  def validate(self, validator):
+    errors = []
+    if validator.is_service_runner():
+      errors.extend(validator.validate_cloud_options(self))
+      errors.extend(validator.validate_gcs_path(self, 'temp_location'))
+      if getattr(self, 'staging_location',
+                 None) or getattr(self, 'temp_location', None) is None:
+        errors.extend(validator.validate_gcs_path(self, 'staging_location'))
+
+    if self.view_as(DebugOptions).dataflow_job_file:
+      if self.view_as(GoogleCloudOptions).template_location:
+        errors.append('--dataflow_job_file and --template_location '
+                      'are mutually exclusive.')
+
+    return errors
+
+
+# Command line options controlling the worker pool configuration.
+# TODO(silviuc): Update description when autoscaling options are in.
+class WorkerOptions(PipelineOptions):
+  """Worker pool configuration options."""
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    parser.add_argument(
+        '--num_workers',
+        type=int,
+        default=None,
+        help=
+        ('Number of workers to use when executing the Dataflow job. If not '
+         'set, the Dataflow service will use a reasonable default.'))
+    parser.add_argument(
+        '--max_num_workers',
+        type=int,
+        default=None,
+        help=
+        ('Maximum number of workers to use when executing the Dataflow job.'))
+    parser.add_argument(
+        '--autoscaling_algorithm',
+        type=str,
+        choices=['NONE', 'THROUGHPUT_BASED'],
+        default=None,  # Meaning unset, distinct from 'NONE' meaning don't scale
+        help=
+        ('If and how to autoscale the workerpool.'))
+    parser.add_argument(
+        '--worker_machine_type',
+        dest='machine_type',
+        default=None,
+        help=('Machine type to create Dataflow worker VMs as. See '
+              'https://cloud.google.com/compute/docs/machine-types '
+              'for a list of valid options. If not set, '
+              'the Dataflow service will choose a reasonable '
+              'default.'))
+    parser.add_argument(
+        '--disk_size_gb',
+        type=int,
+        default=None,
+        help=
+        ('Remote worker disk size, in gigabytes, or 0 to use the default size. '
+         'If not set, the Dataflow service will use a reasonable default.'))
+    parser.add_argument(
+        '--worker_disk_type',
+        dest='disk_type',
+        default=None,
+        help=('Specifies what type of persistent disk should be used.'))
+    parser.add_argument(
+        '--zone',
+        default=None,
+        help=(
+            'GCE availability zone for launching workers. Default is up to the '
+            'Dataflow service.'))
+    parser.add_argument(
+        '--network',
+        default=None,
+        help=(
+            'GCE network for launching workers. Default is up to the Dataflow '
+            'service.'))
+    parser.add_argument(
+        '--worker_harness_container_image',
+        default=None,
+        help=('Docker registry location of container image to use for the '
+              'worker harness. Default is the container for the version of the '
+              'SDK. Note: currently, only approved Google Cloud Dataflow '
+              'container images may be used here.'))
+    parser.add_argument(
+        '--use_public_ips',
+        default=None,
+        help='Whether to assign public IP addresses to the worker machines.')
+
+  def validate(self, validator):
+    errors = []
+    if validator.is_service_runner():
+      errors.extend(
+          validator.validate_optional_argument_positive(self, 'num_workers'))
+    return errors
+
+
+class DebugOptions(PipelineOptions):
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    parser.add_argument('--dataflow_job_file',
+                        default=None,
+                        help='Debug file to write the workflow specification.')
+    parser.add_argument(
+        '--experiment', '--experiments',
+        dest='experiments',
+        action='append',
+        default=None,
+        help=
+        ('Runners may provide a number of experimental features that can be '
+         'enabled with this flag. Please sync with the owners of the runner '
+         'before enabling any experiments.'))
+
+
+class ProfilingOptions(PipelineOptions):
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    parser.add_argument('--profile_cpu',
+                        action='store_true',
+                        help='Enable work item CPU profiling.')
+    parser.add_argument('--profile_memory',
+                        action='store_true',
+                        help='Enable work item heap profiling.')
+    parser.add_argument('--profile_location',
+                        default=None,
+                        help='GCS path for saving profiler data.')
+
+
+class SetupOptions(PipelineOptions):
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    # Options for installing dependencies in the worker.
+    parser.add_argument(
+        '--requirements_file',
+        default=None,
+        help=
+        ('Path to a requirements file containing package dependencies. '
+         'Typically it is produced by a pip freeze command. More details: '
+         'https://pip.pypa.io/en/latest/reference/pip_freeze.html. '
+         'If used, all the packages specified will be downloaded, '
+         'cached (use --requirements_cache to change default location), '
+         'and then staged so that they can be automatically installed in '
+         'workers during startup. The cache is refreshed as needed '
+         'avoiding extra downloads for existing packages. Typically the '
+         'file is named requirements.txt.'))
+    parser.add_argument(
+        '--requirements_cache',
+        default=None,
+        help=
+        ('Path to a folder to cache the packages specified in '
+         'the requirements file using the --requirements_file option.'))
+    parser.add_argument(
+        '--setup_file',
+        default=None,
+        help=
+        ('Path to a setup Python file containing package dependencies. If '
+         'specified, the file\'s containing folder is assumed to have the '
+         'structure required for a setuptools setup package. The file must be '
+         'named setup.py. More details: '
+         'https://pythonhosted.org/an_example_pypi_project/setuptools.html '
+         'During job submission a source distribution will be built and the '
+         'worker will install the resulting package before running any custom '
+         'code.'))
+    parser.add_argument(
+        '--save_main_session',
+        default=False,
+        action='store_true',
+        help=
+        ('Save the main session state so that pickled functions and classes '
+         'defined in __main__ (e.g. interactive session) can be unpickled. '
+         'Some workflows do not need the session state if for instance all '
+         'their functions/classes are defined in proper modules (not __main__)'
+         ' and the modules are importable in the worker. '))
+    parser.add_argument(
+        '--sdk_location',
+        default='default',
+        help=
+        ('Override the default location from where the Beam SDK is downloaded. '
+         'It can be a URL, a GCS path, or a local path to an SDK tarball. '
+         'Workflow submissions will download or copy an SDK tarball from here. '
+         'If set to the string "default", a standard SDK location is used. If '
+         'empty, no SDK is copied.'))
+    parser.add_argument(
+        '--extra_package', '--extra_packages',
+        dest='extra_packages',
+        action='append',
+        default=None,
+        help=
+        ('Local path to a Python package file. The file is expected to be (1) '
+         'a package tarball (".tar") or (2) a compressed package tarball '
+         '(".tar.gz") which can be installed using the "pip install" command '
+         'of the standard pip package. Multiple --extra_package options can '
+         'be specified if more than one package is needed. During job '
+         'submission, the files will be staged in the staging area '
+         '(--staging_location option) and the workers will install them in '
+         'same order they were specified on the command line.'))
+
+
+class TestOptions(PipelineOptions):
+
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    # Options for e2e test pipeline.
+    parser.add_argument(
+        '--on_success_matcher',
+        default=None,
+        help=('Verify state/output of e2e test pipeline. This is pickled '
+              'version of the matcher which should extends '
+              'hamcrest.core.base_matcher.BaseMatcher.'))
+
+  def validate(self, validator):
+    errors = []
+    if self.view_as(TestOptions).on_success_matcher:
+      errors.extend(validator.validate_test_matcher(self, 'on_success_matcher'))
+    return errors
+
+# TODO(silviuc): Add --files_to_stage option.
+# This could potentially replace the --requirements_file and --setup_file.
+
+# TODO(silviuc): Non-standard options. Keep them? If yes, add help too!
+# Remote execution must check that this option is not None.
+
+
+class OptionsContext(object):
+  """Set default pipeline options for pipelines created in this block.
+
+  This is particularly useful for pipelines implicitly created with the
+
+      [python list] | PTransform
+
+  construct.
+
+  Can also be used as a decorator.
+  """
+  overrides = []
+
+  def __init__(self, **options):
+    self.options = options
+
+  def __enter__(self):
+    self.overrides.append(self.options)
+
+  def __exit__(self, *exn_info):
+    self.overrides.pop()
+
+  def __call__(self, f, *args, **kwargs):
+
+    def wrapper(*args, **kwargs):
+      with self:
+        f(*args, **kwargs)
+
+    return wrapper
+
+  @classmethod
+  def augment_options(cls, options):
+    for override in cls.overrides:
+      for name, value in override.items():
+        setattr(options, name, value)
+    return options
diff --git a/sdks/python/apache_beam/utils/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py
similarity index 97%
rename from sdks/python/apache_beam/utils/pipeline_options_test.py
rename to sdks/python/apache_beam/options/pipeline_options_test.py
index 284736a77886..1a644b449e42 100644
--- a/sdks/python/apache_beam/utils/pipeline_options_test.py
+++ b/sdks/python/apache_beam/options/pipeline_options_test.py
@@ -23,9 +23,9 @@
 import hamcrest as hc
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.value_provider import StaticValueProvider
-from apache_beam.utils.value_provider import RuntimeValueProvider
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import RuntimeValueProvider
 
 
 class PipelineOptionsTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/utils/pipeline_options_validator.py b/sdks/python/apache_beam/options/pipeline_options_validator.py
similarity index 94%
rename from sdks/python/apache_beam/utils/pipeline_options_validator.py
rename to sdks/python/apache_beam/options/pipeline_options_validator.py
index d5dc43d31185..5c1ce2ac9b1b 100644
--- a/sdks/python/apache_beam/utils/pipeline_options_validator.py
+++ b/sdks/python/apache_beam/options/pipeline_options_validator.py
@@ -20,13 +20,13 @@
 import re
 
 from apache_beam.internal import pickler
-from apache_beam.utils.pipeline_options import DebugOptions
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import SetupOptions
-from apache_beam.utils.pipeline_options import StandardOptions
-from apache_beam.utils.pipeline_options import TestOptions
-from apache_beam.utils.pipeline_options import TypeOptions
-from apache_beam.utils.pipeline_options import WorkerOptions
+from apache_beam.options.pipeline_options import DebugOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.options.pipeline_options import TestOptions
+from apache_beam.options.pipeline_options import TypeOptions
+from apache_beam.options.pipeline_options import WorkerOptions
 
 
 class PipelineOptionsValidator(object):
diff --git a/sdks/python/apache_beam/utils/pipeline_options_validator_test.py b/sdks/python/apache_beam/options/pipeline_options_validator_test.py
similarity index 98%
rename from sdks/python/apache_beam/utils/pipeline_options_validator_test.py
rename to sdks/python/apache_beam/options/pipeline_options_validator_test.py
index 4b03c1fab80e..28fcbe3e4ab9 100644
--- a/sdks/python/apache_beam/utils/pipeline_options_validator_test.py
+++ b/sdks/python/apache_beam/options/pipeline_options_validator_test.py
@@ -21,8 +21,8 @@
 import unittest
 
 from apache_beam.internal import pickler
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options_validator import PipelineOptionsValidator
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator
 from hamcrest.core.base_matcher import BaseMatcher
 
 
diff --git a/sdks/python/apache_beam/options/value_provider.py b/sdks/python/apache_beam/options/value_provider.py
new file mode 100644
index 000000000000..c00d7bc92bce
--- /dev/null
+++ b/sdks/python/apache_beam/options/value_provider.py
@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A ValueProvider class to implement templates with both statically
+and dynamically provided values.
+"""
+
+from functools import wraps
+
+from apache_beam import error
+
+
+class ValueProvider(object):
+  def is_accessible(self):
+    raise NotImplementedError(
+        'ValueProvider.is_accessible implemented in derived classes'
+    )
+
+  def get(self):
+    raise NotImplementedError(
+        'ValueProvider.get implemented in derived classes'
+    )
+
+
+class StaticValueProvider(ValueProvider):
+  def __init__(self, value_type, value):
+    self.value_type = value_type
+    self.value = value_type(value)
+
+  def is_accessible(self):
+    return True
+
+  def get(self):
+    return self.value
+
+  def __str__(self):
+    return str(self.value)
+
+
+class RuntimeValueProvider(ValueProvider):
+  runtime_options = None
+
+  def __init__(self, option_name, value_type, default_value):
+    self.option_name = option_name
+    self.default_value = default_value
+    self.value_type = value_type
+
+  def is_accessible(self):
+    return RuntimeValueProvider.runtime_options is not None
+
+  def get(self):
+    if RuntimeValueProvider.runtime_options is None:
+      raise error.RuntimeValueProviderError(
+          '%s.get() not called from a runtime context' % self)
+
+    candidate = RuntimeValueProvider.runtime_options.get(self.option_name)
+    if candidate:
+      value = self.value_type(candidate)
+    else:
+      value = self.default_value
+    return value
+
+  # TODO(BEAM-1999): Remove _unused_options_id
+  @classmethod
+  def set_runtime_options(cls, pipeline_options):
+    RuntimeValueProvider.runtime_options = pipeline_options
+
+  def __str__(self):
+    return '%s(option: %s, type: %s, default_value: %s)' % (
+        self.__class__.__name__,
+        self.option_name,
+        self.value_type.__name__,
+        repr(self.default_value)
+    )
+
+
+def check_accessible(value_provider_list):
+  """Check accessibility of a list of ValueProvider objects."""
+  assert isinstance(value_provider_list, list)
+
+  def _check_accessible(fnc):
+    @wraps(fnc)
+    def _f(self, *args, **kwargs):
+      for obj in [getattr(self, vp) for vp in value_provider_list]:
+        if not obj.is_accessible():
+          raise error.RuntimeValueProviderError('%s not accessible' % obj)
+      return fnc(self, *args, **kwargs)
+    return _f
+  return _check_accessible
diff --git a/sdks/python/apache_beam/utils/value_provider_test.py b/sdks/python/apache_beam/options/value_provider_test.py
similarity index 96%
rename from sdks/python/apache_beam/utils/value_provider_test.py
rename to sdks/python/apache_beam/options/value_provider_test.py
index 1b66dd4eb9a2..3a45e8b7d122 100644
--- a/sdks/python/apache_beam/utils/value_provider_test.py
+++ b/sdks/python/apache_beam/options/value_provider_test.py
@@ -19,9 +19,9 @@
 
 import unittest
 
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.value_provider import RuntimeValueProvider
-from apache_beam.utils.value_provider import StaticValueProvider
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.value_provider import RuntimeValueProvider
+from apache_beam.options.value_provider import StaticValueProvider
 
 
 class ValueProviderTests(unittest.TestCase):
diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index 9200363afe1b..ed896c4fd0d1 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -59,11 +59,11 @@
 from apache_beam.runners import PipelineRunner
 from apache_beam.transforms import ptransform
 from apache_beam.typehints import TypeCheckError
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
-from apache_beam.utils.pipeline_options import StandardOptions
-from apache_beam.utils.pipeline_options import TypeOptions
-from apache_beam.utils.pipeline_options_validator import PipelineOptionsValidator
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.options.pipeline_options import TypeOptions
+from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator
 from apache_beam.utils.annotations import deprecated
 
 
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 3332033e5735..796a67b8cbdd 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -43,7 +43,7 @@
 from apache_beam.runners.runner import PipelineState
 from apache_beam.transforms.display import DisplayData
 from apache_beam.typehints import typehints
-from apache_beam.utils.pipeline_options import StandardOptions
+from apache_beam.options.pipeline_options import StandardOptions
 
 
 class DataflowRunner(PipelineRunner):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
index 872dfcd5db60..a9f61a7ce7d7 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
@@ -37,7 +37,7 @@
 from apache_beam.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayDataItem
 from apache_beam.typehints import typehints
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 # Protect against environments where apitools library is not available.
 # pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index e64cf8da3a13..220ceb8ed325 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -42,10 +42,10 @@
 from apache_beam.transforms import cy_combiners
 from apache_beam.transforms.display import DisplayData
 from apache_beam.utils import retry
-from apache_beam.utils.pipeline_options import DebugOptions
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import StandardOptions
-from apache_beam.utils.pipeline_options import WorkerOptions
+from apache_beam.options.pipeline_options import DebugOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import StandardOptions
+from apache_beam.options.pipeline_options import WorkerOptions
 
 
 class Step(object):
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index e9aaacb43cb9..6ed1fb4a0322 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -20,7 +20,7 @@
 from mock import Mock
 
 from apache_beam.metrics.cells import DistributionData
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
 from apache_beam.runners.dataflow.internal.clients import dataflow
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index 782cc44f5c92..186ad3f21a77 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -66,8 +66,8 @@
 from apache_beam.io.filesystems import FileSystems
 from apache_beam.runners.dataflow.internal import names
 from apache_beam.utils import processes
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 # Update this version to the next version whenever there is a change that will
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
index 1ff087bb850e..5eac7d608010 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py
@@ -26,9 +26,9 @@
 from apache_beam.io.filesystems import FileSystems
 from apache_beam.runners.dataflow.internal import dependency
 from apache_beam.runners.dataflow.internal import names
-from apache_beam.utils.pipeline_options import GoogleCloudOptions
-from apache_beam.utils.pipeline_options import PipelineOptions
-from apache_beam.utils.pipeline_options import SetupOptions
+from apache_beam.options.pipeline_options import GoogleCloudOptions
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
 
 
 class SetupTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py
index 5eb0f23265f9..7927219ab5ea 100644
--- a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py
@@ -25,7 +25,7 @@
 
 import apache_beam as beam
 from apache_beam.pipeline import Pipeline
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
 
 # Protect against environments where apitools library is not available.
diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
index 290c7ad73263..4fd1026c8285 100644
--- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
@@ -18,7 +18,7 @@
 """Wrapper of Beam runners that's built for running and verifying e2e tests."""
 
 from apache_beam.internal import pickler
-from apache_beam.utils.pipeline_options import TestOptions, GoogleCloudOptions
+from apache_beam.options.pipeline_options import TestOptions, GoogleCloudOptions
 from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
 
 
diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py
index d5aba5ab2a91..535aac33ed96 100644
--- a/sdks/python/apache_beam/runners/direct/direct_runner.py
+++ b/sdks/python/apache_beam/runners/direct/direct_runner.py
@@ -32,8 +32,8 @@
 from apache_beam.runners.runner import PipelineRunner
 from apache_beam.runners.runner import PipelineState
 from apache_beam.runners.runner import PValueCache
-from apache_beam.utils.pipeline_options import DirectOptions
-from apache_beam.utils.value_provider import RuntimeValueProvider
+from apache_beam.options.pipeline_options import DirectOptions
+from apache_beam.options.value_provider import RuntimeValueProvider
 
 
 class DirectRunner(PipelineRunner):
diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
index 16b3131b8867..f6cdd414eb48 100644
--- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py
+++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
@@ -37,7 +37,7 @@
 from apache_beam.typehints.typecheck import TypeCheckError
 from apache_beam.typehints.typecheck import TypeCheckWrapperDoFn
 from apache_beam.utils import counters
-from apache_beam.utils.pipeline_options import TypeOptions
+from apache_beam.options.pipeline_options import TypeOptions
 
 
 class TransformEvaluatorRegistry(object):
diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
index d273e189e033..c07f27ff0de0 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
@@ -26,6 +26,7 @@
 from apache_beam.internal import pickler
 from apache_beam.io import iobase
 from apache_beam.metrics.execution import MetricsEnvironment
+from apache_beam.options import pipeline_options
 from apache_beam.runners import DataflowRunner
 from apache_beam.runners.dataflow.internal.dependency import _dependency_file_copy
 from apache_beam.runners.dataflow.internal.names import PropertyNames
@@ -39,7 +40,6 @@
 except ImportError:
   from apache_beam.runners.worker import statesampler_fake as statesampler
 from apache_beam.typehints import typehints
-from apache_beam.utils import pipeline_options
 from apache_beam.utils import profiler
 from apache_beam.utils.counters import CounterFactory
 
diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py
index b161cbbd49b1..c61c49f77916 100644
--- a/sdks/python/apache_beam/runners/runner_test.py
+++ b/sdks/python/apache_beam/runners/runner_test.py
@@ -38,7 +38,7 @@
 from apache_beam.runners import create_runner
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 
 class RunnerTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/test_pipeline.py b/sdks/python/apache_beam/test_pipeline.py
index 7d85af919b86..20f4839bdb2b 100644
--- a/sdks/python/apache_beam/test_pipeline.py
+++ b/sdks/python/apache_beam/test_pipeline.py
@@ -23,7 +23,7 @@
 from apache_beam.internal import pickler
 from apache_beam.pipeline import Pipeline
 from apache_beam.runners.runner import PipelineState
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 from nose.plugins.skip import SkipTest
 
 
diff --git a/sdks/python/apache_beam/test_pipeline_test.py b/sdks/python/apache_beam/test_pipeline_test.py
index c4bcc0149809..325cab789518 100644
--- a/sdks/python/apache_beam/test_pipeline_test.py
+++ b/sdks/python/apache_beam/test_pipeline_test.py
@@ -25,7 +25,7 @@
 
 from apache_beam.internal import pickler
 from apache_beam.test_pipeline import TestPipeline
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 
 # A simple matcher that is ued for testing extra options appending.
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index 62a9b97e6319..9367e6f6ff9c 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -49,7 +49,7 @@
 from apache_beam.typehints import WithTypeHints
 from apache_beam.typehints.trivial_inference import element_type
 from apache_beam.utils import urns
-from apache_beam.utils.pipeline_options import TypeOptions
+from apache_beam.options.pipeline_options import TypeOptions
 
 
 # Type variables
diff --git a/sdks/python/apache_beam/transforms/display.py b/sdks/python/apache_beam/transforms/display.py
index 6477db589be8..152f16e2d9fb 100644
--- a/sdks/python/apache_beam/transforms/display.py
+++ b/sdks/python/apache_beam/transforms/display.py
@@ -124,7 +124,7 @@ def create_from_options(cls, pipeline_options):
       ValueError: If the has_display_data argument is not an instance of
         HasDisplayData.
     """
-    from apache_beam.utils.pipeline_options import PipelineOptions
+    from apache_beam.options.pipeline_options import PipelineOptions
     if not isinstance(pipeline_options, PipelineOptions):
       raise ValueError(
           'Element of class {}.{} does not subclass PipelineOptions'
diff --git a/sdks/python/apache_beam/transforms/display_test.py b/sdks/python/apache_beam/transforms/display_test.py
index 7d1130ba1ccf..15f178645bc3 100644
--- a/sdks/python/apache_beam/transforms/display_test.py
+++ b/sdks/python/apache_beam/transforms/display_test.py
@@ -29,7 +29,7 @@
 from apache_beam.transforms.display import HasDisplayData
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.utils.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import PipelineOptions
 
 
 class DisplayDataItemMatcher(BaseMatcher):
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index 706b003d2445..526291c43fc9 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -352,7 +352,7 @@ def __ror__(self, left, label=None):
       deferred = False
       # pylint: disable=wrong-import-order, wrong-import-position
       from apache_beam import pipeline
-      from apache_beam.utils.pipeline_options import PipelineOptions
+      from apache_beam.options.pipeline_options import PipelineOptions
       # pylint: enable=wrong-import-order, wrong-import-position
       p = pipeline.Pipeline(
           'DirectRunner', PipelineOptions(sys.argv))
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index ab9417ec9629..b8b0733622e4 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -40,7 +40,7 @@
 from apache_beam.typehints import with_input_types
 from apache_beam.typehints import with_output_types
 from apache_beam.typehints.typehints_test import TypeHintTestCase
-from apache_beam.utils.pipeline_options import TypeOptions
+from apache_beam.options.pipeline_options import TypeOptions
 
 
 # Disable frequent lint warning due to pipe operator for chaining transforms.
diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
index 32f6a71e0af7..3812fb1270e2 100644
--- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
+++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
@@ -26,7 +26,7 @@
 from apache_beam.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that, equal_to
 from apache_beam.typehints import WithTypeHints
-from apache_beam.utils.pipeline_options import OptionsContext
+from apache_beam.options.pipeline_options import OptionsContext
 
 # These test often construct a pipeline as value | PTransform to test side
 # effects (e.g. errors).
diff --git a/sdks/python/apache_beam/utils/pipeline_options.py b/sdks/python/apache_beam/utils/pipeline_options.py
index e8966d6ca08c..51a17ed4cd53 100644
--- a/sdks/python/apache_beam/utils/pipeline_options.py
+++ b/sdks/python/apache_beam/utils/pipeline_options.py
@@ -15,613 +15,7 @@
 # limitations under the License.
 #
 
-"""Pipeline options obtained from command line parsing."""
+"""For backwards compatibility"""
 
-import argparse
-
-from apache_beam.transforms.display import HasDisplayData
-from apache_beam.utils.value_provider import StaticValueProvider
-from apache_beam.utils.value_provider import RuntimeValueProvider
-from apache_beam.utils.value_provider import ValueProvider
-
-
-def _static_value_provider_of(value_type):
-  """"Helper function to plug a ValueProvider into argparse.
-
-  Args:
-    value_type: the type of the value. Since the type param of argparse's
-                add_argument will always be ValueProvider, we need to
-                preserve the type of the actual value.
-  Returns:
-    A partially constructed StaticValueProvider in the form of a function.
-
-  """
-  def _f(value):
-    _f.func_name = value_type.__name__
-    return StaticValueProvider(value_type, value)
-  return _f
-
-
-class BeamArgumentParser(argparse.ArgumentParser):
-  """An ArgumentParser that supports ValueProvider options.
-
-  Example Usage::
-
-    class TemplateUserOptions(PipelineOptions):
-      @classmethod
-
-      def _add_argparse_args(cls, parser):
-        parser.add_value_provider_argument('--vp-arg1', default='start')
-        parser.add_value_provider_argument('--vp-arg2')
-        parser.add_argument('--non-vp-arg')
-
-  """
-  def add_value_provider_argument(self, *args, **kwargs):
-    """ValueProvider arguments can be either of type keyword or positional.
-    At runtime, even positional arguments will need to be supplied in the
-    key/value form.
-    """
-    # Extract the option name from positional argument ['pos_arg']
-    assert args != () and len(args[0]) >= 1
-    if args[0][0] != '-':
-      option_name = args[0]
-      if kwargs.get('nargs') is None:  # make them optionally templated
-        kwargs['nargs'] = '?'
-    else:
-      # or keyword arguments like [--kw_arg, -k, -w] or [--kw-arg]
-      option_name = [i.replace('--', '') for i in args if i[:2] == '--'][0]
-
-    # reassign the type to make room for using
-    # StaticValueProvider as the type for add_argument
-    value_type = kwargs.get('type') or str
-    kwargs['type'] = _static_value_provider_of(value_type)
-
-    # reassign default to default_value to make room for using
-    # RuntimeValueProvider as the default for add_argument
-    default_value = kwargs.get('default')
-    kwargs['default'] = RuntimeValueProvider(
-        option_name=option_name,
-        value_type=value_type,
-        default_value=default_value
-    )
-
-    # have add_argument do most of the work
-    self.add_argument(*args, **kwargs)
-
-
-class PipelineOptions(HasDisplayData):
-  """Pipeline options class used as container for command line options.
-
-  The class is essentially a wrapper over the standard argparse Python module
-  (see https://docs.python.org/3/library/argparse.html).  To define one option
-  or a group of options you subclass from PipelineOptions::
-
-    class XyzOptions(PipelineOptions):
-
-      @classmethod
-      def _add_argparse_args(cls, parser):
-        parser.add_argument('--abc', default='start')
-        parser.add_argument('--xyz', default='end')
-
-  The arguments for the add_argument() method are exactly the ones
-  described in the argparse public documentation.
-
-  Pipeline objects require an options object during initialization.
-  This is obtained simply by initializing an options class as defined above::
-
-    p = Pipeline(options=XyzOptions())
-    if p.options.xyz == 'end':
-      raise ValueError('Option xyz has an invalid value.')
-
-  By default the options classes will use command line arguments to initialize
-  the options.
-  """
-  def __init__(self, flags=None, **kwargs):
-    """Initialize an options class.
-
-    The initializer will traverse all subclasses, add all their argparse
-    arguments and then parse the command line specified by flags or by default
-    the one obtained from sys.argv.
-
-    The subclasses are not expected to require a redefinition of __init__.
-
-    Args:
-      flags: An iterable of command line arguments to be used. If not specified
-        then sys.argv will be used as input for parsing arguments.
-
-      **kwargs: Add overrides for arguments passed in flags.
-    """
-    self._flags = flags
-    self._all_options = kwargs
-    parser = BeamArgumentParser()
-
-    for cls in type(self).mro():
-      if cls == PipelineOptions:
-        break
-      elif '_add_argparse_args' in cls.__dict__:
-        cls._add_argparse_args(parser)
-    # The _visible_options attribute will contain only those options from the
-    # flags (i.e., command line) that can be recognized. The _all_options
-    # field contains additional overrides.
-    self._visible_options, _ = parser.parse_known_args(flags)
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    # Override this in subclasses to provide options.
-    pass
-
-  @classmethod
-  def from_dictionary(cls, options):
-    """Returns a PipelineOptions from a dictionary of arguments.
-
-    Args:
-      options: Dictinary of argument value pairs.
-
-    Returns:
-      A PipelineOptions object representing the given arguments.
-    """
-    flags = []
-    for k, v in options.iteritems():
-      if isinstance(v, bool):
-        if v:
-          flags.append('--%s' % k)
-      else:
-        flags.append('--%s=%s' % (k, v))
-
-    return cls(flags)
-
-  def get_all_options(self, drop_default=False):
-    """Returns a dictionary of all defined arguments.
-
-    Returns a dictionary of all defined arguments (arguments that are defined in
-    any subclass of PipelineOptions) into a dictionary.
-
-    Args:
-      drop_default: If set to true, options that are equal to their default
-        values, are not returned as part of the result dictionary.
-
-    Returns:
-      Dictionary of all args and values.
-    """
-
-    # TODO(BEAM-1319): PipelineOption sub-classes in the main session might be
-    # repeated. Pick last unique instance of each subclass to avoid conflicts.
-    subset = {}
-    parser = BeamArgumentParser()
-    for cls in PipelineOptions.__subclasses__():
-      subset[str(cls)] = cls
-    for cls in subset.values():
-      cls._add_argparse_args(parser)  # pylint: disable=protected-access
-    known_args, _ = parser.parse_known_args(self._flags)
-    result = vars(known_args)
-
-    # Apply the overrides if any
-    for k in result.keys():
-      if k in self._all_options:
-        result[k] = self._all_options[k]
-      if (drop_default and
-          parser.get_default(k) == result[k] and
-          not isinstance(parser.get_default(k), ValueProvider)):
-        del result[k]
-
-    return result
-
-  def display_data(self):
-    return self.get_all_options(True)
-
-  def view_as(self, cls):
-    view = cls(self._flags)
-    view._all_options = self._all_options
-    return view
-
-  def _visible_option_list(self):
-    return sorted(option
-                  for option in dir(self._visible_options) if option[0] != '_')
-
-  def __dir__(self):
-    return sorted(dir(type(self)) + self.__dict__.keys() +
-                  self._visible_option_list())
-
-  def __getattr__(self, name):
-    # Special methods which may be accessed before the object is
-    # fully constructed (e.g. in unpickling).
-    if name[:2] == name[-2:] == '__':
-      return object.__getattribute__(self, name)
-    elif name in self._visible_option_list():
-      return self._all_options.get(name, getattr(self._visible_options, name))
-    else:
-      raise AttributeError("'%s' object has no attribute '%s'" %
-                           (type(self).__name__, name))
-
-  def __setattr__(self, name, value):
-    if name in ('_flags', '_all_options', '_visible_options'):
-      super(PipelineOptions, self).__setattr__(name, value)
-    elif name in self._visible_option_list():
-      self._all_options[name] = value
-    else:
-      raise AttributeError("'%s' object has no attribute '%s'" %
-                           (type(self).__name__, name))
-
-  def __str__(self):
-    return '%s(%s)' % (type(self).__name__,
-                       ', '.join('%s=%s' % (option, getattr(self, option))
-                                 for option in self._visible_option_list()))
-
-
-class StandardOptions(PipelineOptions):
-
-  DEFAULT_RUNNER = 'DirectRunner'
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    parser.add_argument(
-        '--runner',
-        help=('Pipeline runner used to execute the workflow. Valid values are '
-              'DirectRunner, DataflowRunner.'))
-    # Whether to enable streaming mode.
-    parser.add_argument('--streaming',
-                        default=False,
-                        action='store_true',
-                        help='Whether to enable streaming mode.')
-
-  # TODO(BEAM-1265): Remove this error, once at least one runner supports
-  # streaming pipelines.
-  def validate(self, validator):
-    errors = []
-    if self.view_as(StandardOptions).streaming:
-      errors.append('Streaming pipelines are not supported.')
-    return errors
-
-
-class TypeOptions(PipelineOptions):
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    # TODO(laolu): Add a type inferencing option here once implemented.
-    parser.add_argument('--type_check_strictness',
-                        default='DEFAULT_TO_ANY',
-                        choices=['ALL_REQUIRED', 'DEFAULT_TO_ANY'],
-                        help='The level of exhaustive manual type-hint '
-                        'annotation required')
-    parser.add_argument('--no_pipeline_type_check',
-                        dest='pipeline_type_check',
-                        action='store_false',
-                        help='Disable type checking at pipeline construction '
-                        'time')
-    parser.add_argument('--runtime_type_check',
-                        default=False,
-                        action='store_true',
-                        help='Enable type checking at pipeline execution '
-                        'time. NOTE: only supported with the '
-                        'DirectRunner')
-
-
-class DirectOptions(PipelineOptions):
-  """DirectRunner-specific execution options."""
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    parser.add_argument(
-        '--no_direct_runner_use_stacked_bundle',
-        action='store_false',
-        dest='direct_runner_use_stacked_bundle',
-        help='DirectRunner uses stacked WindowedValues within a Bundle for '
-        'memory optimization. Set --no_direct_runner_use_stacked_bundle to '
-        'avoid it.')
-
-
-class GoogleCloudOptions(PipelineOptions):
-  """Google Cloud Dataflow service execution options."""
-
-  BIGQUERY_API_SERVICE = 'bigquery.googleapis.com'
-  COMPUTE_API_SERVICE = 'compute.googleapis.com'
-  STORAGE_API_SERVICE = 'storage.googleapis.com'
-  DATAFLOW_ENDPOINT = 'https://dataflow.googleapis.com'
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    parser.add_argument(
-        '--dataflow_endpoint',
-        default=cls.DATAFLOW_ENDPOINT,
-        help=
-        ('The URL for the Dataflow API. If not set, the default public URL '
-         'will be used.'))
-    # Remote execution must check that this option is not None.
-    parser.add_argument('--project',
-                        default=None,
-                        help='Name of the Cloud project owning the Dataflow '
-                        'job.')
-    # Remote execution must check that this option is not None.
-    parser.add_argument('--job_name',
-                        default=None,
-                        help='Name of the Cloud Dataflow job.')
-    # Remote execution must check that this option is not None.
-    parser.add_argument('--staging_location',
-                        default=None,
-                        help='GCS path for staging code packages needed by '
-                        'workers.')
-    # Remote execution must check that this option is not None.
-    # If staging_location is not set, it defaults to temp_location.
-    parser.add_argument('--temp_location',
-                        default=None,
-                        help='GCS path for saving temporary workflow jobs.')
-    # The Cloud Dataflow service does not yet honor this setting. However, once
-    # service support is added then users of this SDK will be able to control
-    # the region. Default is up to the Dataflow service. See
-    # https://cloud.google.com/compute/docs/regions-zones/regions-zones for a
-    # list of valid options/
-    parser.add_argument('--region',
-                        default='us-central1',
-                        help='The Google Compute Engine region for creating '
-                        'Dataflow job.')
-    parser.add_argument('--service_account_email',
-                        default=None,
-                        help='Identity to run virtual machines as.')
-    parser.add_argument('--no_auth', dest='no_auth', type=bool, default=False)
-    # Option to run templated pipelines
-    parser.add_argument('--template_location',
-                        default=None,
-                        help='Save job to specified local or GCS location.')
-
-  def validate(self, validator):
-    errors = []
-    if validator.is_service_runner():
-      errors.extend(validator.validate_cloud_options(self))
-      errors.extend(validator.validate_gcs_path(self, 'temp_location'))
-      if getattr(self, 'staging_location',
-                 None) or getattr(self, 'temp_location', None) is None:
-        errors.extend(validator.validate_gcs_path(self, 'staging_location'))
-
-    if self.view_as(DebugOptions).dataflow_job_file:
-      if self.view_as(GoogleCloudOptions).template_location:
-        errors.append('--dataflow_job_file and --template_location '
-                      'are mutually exclusive.')
-
-    return errors
-
-
-# Command line options controlling the worker pool configuration.
-# TODO(silviuc): Update description when autoscaling options are in.
-class WorkerOptions(PipelineOptions):
-  """Worker pool configuration options."""
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    parser.add_argument(
-        '--num_workers',
-        type=int,
-        default=None,
-        help=
-        ('Number of workers to use when executing the Dataflow job. If not '
-         'set, the Dataflow service will use a reasonable default.'))
-    parser.add_argument(
-        '--max_num_workers',
-        type=int,
-        default=None,
-        help=
-        ('Maximum number of workers to use when executing the Dataflow job.'))
-    parser.add_argument(
-        '--autoscaling_algorithm',
-        type=str,
-        choices=['NONE', 'THROUGHPUT_BASED'],
-        default=None,  # Meaning unset, distinct from 'NONE' meaning don't scale
-        help=
-        ('If and how to autoscale the workerpool.'))
-    parser.add_argument(
-        '--worker_machine_type',
-        dest='machine_type',
-        default=None,
-        help=('Machine type to create Dataflow worker VMs as. See '
-              'https://cloud.google.com/compute/docs/machine-types '
-              'for a list of valid options. If not set, '
-              'the Dataflow service will choose a reasonable '
-              'default.'))
-    parser.add_argument(
-        '--disk_size_gb',
-        type=int,
-        default=None,
-        help=
-        ('Remote worker disk size, in gigabytes, or 0 to use the default size. '
-         'If not set, the Dataflow service will use a reasonable default.'))
-    parser.add_argument(
-        '--worker_disk_type',
-        dest='disk_type',
-        default=None,
-        help=('Specifies what type of persistent disk should be used.'))
-    parser.add_argument(
-        '--zone',
-        default=None,
-        help=(
-            'GCE availability zone for launching workers. Default is up to the '
-            'Dataflow service.'))
-    parser.add_argument(
-        '--network',
-        default=None,
-        help=(
-            'GCE network for launching workers. Default is up to the Dataflow '
-            'service.'))
-    parser.add_argument(
-        '--worker_harness_container_image',
-        default=None,
-        help=('Docker registry location of container image to use for the '
-              'worker harness. Default is the container for the version of the '
-              'SDK. Note: currently, only approved Google Cloud Dataflow '
-              'container images may be used here.'))
-    parser.add_argument(
-        '--use_public_ips',
-        default=None,
-        help='Whether to assign public IP addresses to the worker machines.')
-
-  def validate(self, validator):
-    errors = []
-    if validator.is_service_runner():
-      errors.extend(
-          validator.validate_optional_argument_positive(self, 'num_workers'))
-    return errors
-
-
-class DebugOptions(PipelineOptions):
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    parser.add_argument('--dataflow_job_file',
-                        default=None,
-                        help='Debug file to write the workflow specification.')
-    parser.add_argument(
-        '--experiment', '--experiments',
-        dest='experiments',
-        action='append',
-        default=None,
-        help=
-        ('Runners may provide a number of experimental features that can be '
-         'enabled with this flag. Please sync with the owners of the runner '
-         'before enabling any experiments.'))
-
-
-class ProfilingOptions(PipelineOptions):
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    parser.add_argument('--profile_cpu',
-                        action='store_true',
-                        help='Enable work item CPU profiling.')
-    parser.add_argument('--profile_memory',
-                        action='store_true',
-                        help='Enable work item heap profiling.')
-    parser.add_argument('--profile_location',
-                        default=None,
-                        help='GCS path for saving profiler data.')
-
-
-class SetupOptions(PipelineOptions):
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    # Options for installing dependencies in the worker.
-    parser.add_argument(
-        '--requirements_file',
-        default=None,
-        help=
-        ('Path to a requirements file containing package dependencies. '
-         'Typically it is produced by a pip freeze command. More details: '
-         'https://pip.pypa.io/en/latest/reference/pip_freeze.html. '
-         'If used, all the packages specified will be downloaded, '
-         'cached (use --requirements_cache to change default location), '
-         'and then staged so that they can be automatically installed in '
-         'workers during startup. The cache is refreshed as needed '
-         'avoiding extra downloads for existing packages. Typically the '
-         'file is named requirements.txt.'))
-    parser.add_argument(
-        '--requirements_cache',
-        default=None,
-        help=
-        ('Path to a folder to cache the packages specified in '
-         'the requirements file using the --requirements_file option.'))
-    parser.add_argument(
-        '--setup_file',
-        default=None,
-        help=
-        ('Path to a setup Python file containing package dependencies. If '
-         'specified, the file\'s containing folder is assumed to have the '
-         'structure required for a setuptools setup package. The file must be '
-         'named setup.py. More details: '
-         'https://pythonhosted.org/an_example_pypi_project/setuptools.html '
-         'During job submission a source distribution will be built and the '
-         'worker will install the resulting package before running any custom '
-         'code.'))
-    parser.add_argument(
-        '--save_main_session',
-        default=False,
-        action='store_true',
-        help=
-        ('Save the main session state so that pickled functions and classes '
-         'defined in __main__ (e.g. interactive session) can be unpickled. '
-         'Some workflows do not need the session state if for instance all '
-         'their functions/classes are defined in proper modules (not __main__)'
-         ' and the modules are importable in the worker. '))
-    parser.add_argument(
-        '--sdk_location',
-        default='default',
-        help=
-        ('Override the default location from where the Beam SDK is downloaded. '
-         'It can be a URL, a GCS path, or a local path to an SDK tarball. '
-         'Workflow submissions will download or copy an SDK tarball from here. '
-         'If set to the string "default", a standard SDK location is used. If '
-         'empty, no SDK is copied.'))
-    parser.add_argument(
-        '--extra_package', '--extra_packages',
-        dest='extra_packages',
-        action='append',
-        default=None,
-        help=
-        ('Local path to a Python package file. The file is expected to be (1) '
-         'a package tarball (".tar") or (2) a compressed package tarball '
-         '(".tar.gz") which can be installed using the "pip install" command '
-         'of the standard pip package. Multiple --extra_package options can '
-         'be specified if more than one package is needed. During job '
-         'submission, the files will be staged in the staging area '
-         '(--staging_location option) and the workers will install them in '
-         'same order they were specified on the command line.'))
-
-
-class TestOptions(PipelineOptions):
-
-  @classmethod
-  def _add_argparse_args(cls, parser):
-    # Options for e2e test pipeline.
-    parser.add_argument(
-        '--on_success_matcher',
-        default=None,
-        help=('Verify state/output of e2e test pipeline. This is pickled '
-              'version of the matcher which should extends '
-              'hamcrest.core.base_matcher.BaseMatcher.'))
-
-  def validate(self, validator):
-    errors = []
-    if self.view_as(TestOptions).on_success_matcher:
-      errors.extend(validator.validate_test_matcher(self, 'on_success_matcher'))
-    return errors
-
-# TODO(silviuc): Add --files_to_stage option.
-# This could potentially replace the --requirements_file and --setup_file.
-
-# TODO(silviuc): Non-standard options. Keep them? If yes, add help too!
-# Remote execution must check that this option is not None.
-
-
-class OptionsContext(object):
-  """Set default pipeline options for pipelines created in this block.
-
-  This is particularly useful for pipelines implicitly created with the
-
-      [python list] | PTransform
-
-  construct.
-
-  Can also be used as a decorator.
-  """
-  overrides = []
-
-  def __init__(self, **options):
-    self.options = options
-
-  def __enter__(self):
-    self.overrides.append(self.options)
-
-  def __exit__(self, *exn_info):
-    self.overrides.pop()
-
-  def __call__(self, f, *args, **kwargs):
-
-    def wrapper(*args, **kwargs):
-      with self:
-        f(*args, **kwargs)
-
-    return wrapper
-
-  @classmethod
-  def augment_options(cls, options):
-    for override in cls.overrides:
-      for name, value in override.items():
-        setattr(options, name, value)
-    return options
+# pylint: disable=unused-import
+from apache_beam.options.pipeline_options import *
diff --git a/sdks/python/apache_beam/utils/value_provider.py b/sdks/python/apache_beam/utils/value_provider.py
index c00d7bc92bce..46f39aef7671 100644
--- a/sdks/python/apache_beam/utils/value_provider.py
+++ b/sdks/python/apache_beam/utils/value_provider.py
@@ -15,89 +15,7 @@
 # limitations under the License.
 #
 
-"""A ValueProvider class to implement templates with both statically
-and dynamically provided values.
-"""
+"""For backwards compatibility"""
 
-from functools import wraps
-
-from apache_beam import error
-
-
-class ValueProvider(object):
-  def is_accessible(self):
-    raise NotImplementedError(
-        'ValueProvider.is_accessible implemented in derived classes'
-    )
-
-  def get(self):
-    raise NotImplementedError(
-        'ValueProvider.get implemented in derived classes'
-    )
-
-
-class StaticValueProvider(ValueProvider):
-  def __init__(self, value_type, value):
-    self.value_type = value_type
-    self.value = value_type(value)
-
-  def is_accessible(self):
-    return True
-
-  def get(self):
-    return self.value
-
-  def __str__(self):
-    return str(self.value)
-
-
-class RuntimeValueProvider(ValueProvider):
-  runtime_options = None
-
-  def __init__(self, option_name, value_type, default_value):
-    self.option_name = option_name
-    self.default_value = default_value
-    self.value_type = value_type
-
-  def is_accessible(self):
-    return RuntimeValueProvider.runtime_options is not None
-
-  def get(self):
-    if RuntimeValueProvider.runtime_options is None:
-      raise error.RuntimeValueProviderError(
-          '%s.get() not called from a runtime context' % self)
-
-    candidate = RuntimeValueProvider.runtime_options.get(self.option_name)
-    if candidate:
-      value = self.value_type(candidate)
-    else:
-      value = self.default_value
-    return value
-
-  # TODO(BEAM-1999): Remove _unused_options_id
-  @classmethod
-  def set_runtime_options(cls, pipeline_options):
-    RuntimeValueProvider.runtime_options = pipeline_options
-
-  def __str__(self):
-    return '%s(option: %s, type: %s, default_value: %s)' % (
-        self.__class__.__name__,
-        self.option_name,
-        self.value_type.__name__,
-        repr(self.default_value)
-    )
-
-
-def check_accessible(value_provider_list):
-  """Check accessibility of a list of ValueProvider objects."""
-  assert isinstance(value_provider_list, list)
-
-  def _check_accessible(fnc):
-    @wraps(fnc)
-    def _f(self, *args, **kwargs):
-      for obj in [getattr(self, vp) for vp in value_provider_list]:
-        if not obj.is_accessible():
-          raise error.RuntimeValueProviderError('%s not accessible' % obj)
-      return fnc(self, *args, **kwargs)
-    return _f
-  return _check_accessible
+# pylint: disable=unused-import
+from apache_beam.options.value_provider import *

From d3fc7d18a4a3c394d32d85ef34c08a85712a8b18 Mon Sep 17 00:00:00 2001
From: Reuven Lax <relax@google.com>
Date: Mon, 8 May 2017 09:06:55 -0700
Subject: [PATCH 116/387] TableDescription is allowed to be null.

---
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java  |  7 +++----
 .../bigquery/DynamicDestinationsHelpers.java  |  3 ++-
 .../sdk/io/gcp/bigquery/TableDestination.java | 13 +++++++++---
 .../gcp/bigquery/TableDestinationCoder.java   | 19 +++++++++--------
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java   | 21 ++++++++++---------
 5 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 304864a7ca85..8fb05ffa8c16 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -641,7 +641,6 @@ static List<ResourceId> getExtractFilePaths(String extractDestinationDir, Job ex
   public static <T> Write<T> write() {
     return new AutoValue_BigQueryIO_Write.Builder<T>()
         .setValidate(true)
-        .setTableDescription("")
         .setBigQueryServices(new BigQueryServicesImpl())
         .setCreateDisposition(Write.CreateDisposition.CREATE_IF_NEEDED)
         .setWriteDisposition(Write.WriteDisposition.WRITE_EMPTY)
@@ -684,7 +683,7 @@ public abstract static class Write<T> extends PTransform<PCollection<T>, WriteRe
     abstract CreateDisposition getCreateDisposition();
     abstract WriteDisposition getWriteDisposition();
     /** Table description. Default is empty. */
-    abstract String getTableDescription();
+    @Nullable abstract String getTableDescription();
     /** An option to indicate if table validation is desired. Default is true. */
     abstract boolean getValidate();
     abstract BigQueryServices getBigQueryServices();
@@ -1027,8 +1026,8 @@ public void populateDisplayData(DisplayData.Builder builder) {
                   .withLabel("Table WriteDisposition"))
           .addIfNotDefault(DisplayData.item("validation", getValidate())
               .withLabel("Validation Enabled"), true)
-          .addIfNotDefault(DisplayData.item("tableDescription", getTableDescription())
-                  .withLabel("Table Description"), "");
+          .addIfNotNull(DisplayData.item("tableDescription", getTableDescription())
+                  .withLabel("Table Description"));
     }
 
     /**
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java
index 72a331491604..530e2b6feb6d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java
@@ -40,9 +40,10 @@ class DynamicDestinationsHelpers {
    */
   static class ConstantTableDestinations<T> extends DynamicDestinations<T, TableDestination> {
     private final ValueProvider<String> tableSpec;
+    @Nullable
     private final String tableDescription;
 
-    ConstantTableDestinations(ValueProvider<String> tableSpec, String tableDescription) {
+    ConstantTableDestinations(ValueProvider<String> tableSpec, @Nullable String tableDescription) {
       this.tableSpec = tableSpec;
       this.tableDescription = tableDescription;
     }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
index 7a82c54e277a..ecf35d8d80e7 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
@@ -21,6 +21,7 @@
 import com.google.api.services.bigquery.model.TableReference;
 import java.io.Serializable;
 import java.util.Objects;
+import javax.annotation.Nullable;
 
 /**
  * Encapsulates a BigQuery table destination.
@@ -28,15 +29,16 @@
 public class TableDestination implements Serializable {
   private static final long serialVersionUID = 1L;
   private final String tableSpec;
+  @Nullable
   private final String tableDescription;
 
 
-  public TableDestination(String tableSpec, String tableDescription) {
+  public TableDestination(String tableSpec, @Nullable String tableDescription) {
     this.tableSpec = tableSpec;
     this.tableDescription = tableDescription;
   }
 
-  public TableDestination(TableReference tableReference, String tableDescription) {
+  public TableDestination(TableReference tableReference, @Nullable String tableDescription) {
     this.tableSpec = BigQueryHelpers.toTableSpec(tableReference);
     this.tableDescription = tableDescription;
   }
@@ -49,13 +51,18 @@ public TableReference getTableReference() {
     return BigQueryHelpers.parseTableSpec(tableSpec);
   }
 
+  @Nullable
   public String getTableDescription() {
     return tableDescription;
   }
 
   @Override
   public String toString() {
-    return "tableSpec: " + tableSpec + " tableDescription: " + tableDescription;
+    String toString = "tableSpec: " + tableSpec;
+    if (tableDescription != null) {
+      toString += " tableDescription: " + tableDescription;
+    }
+    return toString;
   }
 
   @Override
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
index 3059e2aecde2..01bc558214fa 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
@@ -22,13 +22,16 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 
 /** A coder for {@link TableDestination} objects. */
 public class TableDestinationCoder extends AtomicCoder<TableDestination> {
   private static final TableDestinationCoder INSTANCE = new TableDestinationCoder();
-  private static final StringUtf8Coder stringCoder = StringUtf8Coder.of();
+  private static final Coder<String> tableSpecCoder = StringUtf8Coder.of();
+  private static final Coder<String> tableDescriptionCoder = NullableCoder.of(StringUtf8Coder.of());
 
   public static TableDestinationCoder of() {
     return INSTANCE;
@@ -40,19 +43,17 @@ public void encode(TableDestination value, OutputStream outStream, Context conte
     if (value == null) {
       throw new CoderException("cannot encode a null value");
     }
-    stringCoder.encode(value.getTableSpec(), outStream, context.nested());
-    stringCoder.encode(value.getTableDescription(), outStream, context.nested());
+    tableSpecCoder.encode(value.getTableSpec(), outStream, context.nested());
+    tableDescriptionCoder.encode(value.getTableDescription(), outStream, context);
   }
 
   @Override
   public TableDestination decode(InputStream inStream, Context context) throws IOException {
-    return new TableDestination(
-        stringCoder.decode(inStream, context.nested()),
-        stringCoder.decode(inStream, context.nested()));
+    String tableSpec = tableSpecCoder.decode(inStream, context.nested());
+    String tableDescription = tableDescriptionCoder.decode(inStream, context);
+    return new TableDestination(tableSpec, tableDescription);
   }
 
   @Override
-  public void verifyDeterministic() throws NonDeterministicException {
-    return;
-  }
+  public void verifyDeterministic() throws NonDeterministicException {}
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 6c15f8736377..0d3f0003fb1f 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -595,6 +595,7 @@ public void testWrite() throws Exception {
         new TableRow().set("name", "c").set("number", 3))
         .withCoder(TableRowJsonCoder.of()))
     .apply(BigQueryIO.writeTableRows().to("dataset-id.table-id")
+        .withTableDescription(null)
         .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
         .withSchema(new TableSchema().setFields(
             ImmutableList.of(
@@ -981,7 +982,7 @@ public void testBuildWrite() {
             BigQueryIO.writeTableRows().to("foo.com:project:somedataset.sometable");
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, "");
+        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, null);
   }
 
   @Test
@@ -999,7 +1000,7 @@ public void testBuildWriteWithoutValidation() {
         null,
         CreateDisposition.CREATE_IF_NEEDED,
         WriteDisposition.WRITE_EMPTY,
-        "",
+        null,
         false);
   }
 
@@ -1010,7 +1011,7 @@ public void testBuildWriteDefaultProject() {
     checkWriteObject(
         write, null, "somedataset", "sometable",
         null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY,
-        "");
+        null);
   }
 
   @Test
@@ -1022,7 +1023,7 @@ public void testBuildWriteWithTableReference() {
     BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows().to(table);
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, "");
+        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, null);
   }
 
   @Test
@@ -1032,7 +1033,7 @@ public void testBuildWriteWithSchema() {
         BigQueryIO.<TableRow>write().to("foo.com:project:somedataset.sometable").withSchema(schema);
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        schema, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, "");
+        schema, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, null);
   }
 
   @Test
@@ -1042,7 +1043,7 @@ public void testBuildWriteWithCreateDispositionNever() {
         .withCreateDisposition(CreateDisposition.CREATE_NEVER);
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        null, CreateDisposition.CREATE_NEVER, WriteDisposition.WRITE_EMPTY, "");
+        null, CreateDisposition.CREATE_NEVER, WriteDisposition.WRITE_EMPTY, null);
   }
 
   @Test
@@ -1052,7 +1053,7 @@ public void testBuildWriteWithCreateDispositionIfNeeded() {
         .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED);
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, "");
+        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, null);
   }
 
   @Test
@@ -1062,7 +1063,7 @@ public void testBuildWriteWithWriteDispositionTruncate() {
         .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE);
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_TRUNCATE, "");
+        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_TRUNCATE, null);
   }
 
   @Test
@@ -1072,7 +1073,7 @@ public void testBuildWriteWithWriteDispositionAppend() {
         .withWriteDisposition(WriteDisposition.WRITE_APPEND);
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_APPEND, "");
+        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_APPEND, null);
   }
 
   @Test
@@ -1082,7 +1083,7 @@ public void testBuildWriteWithWriteDispositionEmpty() {
         .withWriteDisposition(WriteDisposition.WRITE_EMPTY);
     checkWriteObject(
         write, "foo.com:project", "somedataset", "sometable",
-        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, "");
+        null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY, null);
   }
 
   @Test

From 1fd15bd7441e1be72feb3f9535ffe536d730a531 Mon Sep 17 00:00:00 2001
From: Reuven Lax <relax@google.com>
Date: Sun, 7 May 2017 11:50:08 -0700
Subject: [PATCH 117/387] Shortens PubsubIO names for reading/writing messages.

---
 .../java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index ac6cb44be291..048fded462b2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -435,7 +435,7 @@ private static <T> Read<T> read() {
    * messages will only contain a {@link PubsubMessage#getPayload() payload}, but no {@link
    * PubsubMessage#getAttributeMap() attributes}.
    */
-  public static Read<PubsubMessage> readPubsubMessagesWithoutAttributes() {
+  public static Read<PubsubMessage> readMessages() {
     return new AutoValue_PubsubIO_Read.Builder<PubsubMessage>()
         .setCoder(PubsubMessagePayloadOnlyCoder.of())
         .setParseFn(new IdentityMessageFn())
@@ -448,7 +448,7 @@ public static Read<PubsubMessage> readPubsubMessagesWithoutAttributes() {
    * messages will contain both a {@link PubsubMessage#getPayload() payload} and {@link
    * PubsubMessage#getAttributeMap() attributes}.
    */
-  public static Read<PubsubMessage> readPubsubMessagesWithAttributes() {
+  public static Read<PubsubMessage> readMessagesWithAttributes() {
     return new AutoValue_PubsubIO_Read.Builder<PubsubMessage>()
         .setCoder(PubsubMessageWithAttributesCoder.of())
         .setParseFn(new IdentityMessageFn())
@@ -495,7 +495,7 @@ private static <T> Write<T> write() {
   }
 
   /** Returns A {@link PTransform} that writes to a Google Cloud Pub/Sub stream. */
-  public static Write<PubsubMessage> writePubsubMessages() {
+  public static Write<PubsubMessage> writeMessages() {
     return PubsubIO.<PubsubMessage>write().withFormatFn(new IdentityMessageFn());
   }
 

From b7635aadc2532b5c3bdc38f070865091ebf901c1 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Mon, 8 May 2017 13:42:02 -0700
Subject: [PATCH 118/387] Remove apex README.md. This information is already in
 the website in quickstart and apex runner pages.
 https://github.com/apache/beam-site/pull/232 moves small bits of missing
 content.

---
 runners/apex/README.md | 76 ------------------------------------------
 1 file changed, 76 deletions(-)
 delete mode 100644 runners/apex/README.md

diff --git a/runners/apex/README.md b/runners/apex/README.md
deleted file mode 100644
index b9bc74f75d9b..000000000000
--- a/runners/apex/README.md
+++ /dev/null
@@ -1,76 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-Apex Beam Runner ﴾Apex‐Runner﴿
-=============================
-
-Apex‐Runner is a Runner for Apache Beam which executes Beam pipelines with Apache Apex as underlying engine. The runner has broad support for the Beam model and supports streaming and batch pipelines. 
-
-[Apache Apex](http://apex.apache.org/) is a stream processing platform and framework for low-latency, high-throughput and fault-tolerant analytics applications on Apache Hadoop. Apex is Java based and also provides its own API for application development (native compositional and declarative Java API, SQL) with a comprehensive [operator library](https://github.com/apache/apex-malhar). Apex has a unified streaming architecture and can be used for real-time and batch processing. With its stateful stream processing architecture Apex can support all of the concepts in the Beam model (event time, triggers, watermarks etc.).
-
-##Status
-
-Apex-Runner is relatively new. It is fully functional and can currently be used to run pipelines in embedded mode. It does not take advantage of all the performance and scalability that Apex can deliver. This is expected to be addressed with upcoming work, leveraging features like incremental checkpointing, partitioning and operator affinity from Apex. Please see [JIRA](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20component%20%3D%20runner-apex%20AND%20resolution%20%3D%20Unresolved) and we welcome contributions!
-
-##Getting Started
-
-The following shows how to run the WordCount example that is provided with the source code on Apex (the example is identical with the one provided as part of the Beam examples). 
-
-###Installing Beam
-
-To get the latest version of Beam with Apex-Runner, first clone the Beam repository:
-
-```
-git clone https://github.com/apache/beam
-```
-
-Then switch to the newly created directory and run Maven to build the Apache Beam:
-
-```
-cd beam
-mvn clean install -DskipTests
-```
-
-Now Apache Beam and the Apex Runner are installed in your local Maven repository.
-
-###Running an Example
-
-Download something to count:
-
-```
-curl http://www.gutenberg.org/cache/epub/1128/pg1128.txt > /tmp/kinglear.txt
-```
-
-Run the pipeline, using the Apex runner:
-
-```
-cd examples/java
-mvn exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount -Dexec.args="--inputFile=/tmp/kinglear.txt --output=/tmp/wordcounts.txt --runner=ApexRunner" -Papex-runner
-```
-
-Once completed, there will be multiple output files with the base name given above:
-
-```
-$ ls /tmp/out-*
-/tmp/out-00000-of-00003  /tmp/out-00001-of-00003  /tmp/out-00002-of-00003
-```
-
-##Running pipelines on an Apex YARN cluster
-
-Coming soon.

From fba4c010ce90db8b5cdac489d47126ca3ee8db65 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Mon, 8 May 2017 15:00:33 -0700
Subject: [PATCH 119/387] Remove readme.md files that are covered in web site

---
 runners/spark/README.md                      | 124 -------------------
 sdks/java/javadoc/README.md                  |  23 ----
 sdks/python/apache_beam/tests/data/README.md |  20 ---
 3 files changed, 167 deletions(-)
 delete mode 100644 runners/spark/README.md
 delete mode 100644 sdks/java/javadoc/README.md
 delete mode 100644 sdks/python/apache_beam/tests/data/README.md

diff --git a/runners/spark/README.md b/runners/spark/README.md
deleted file mode 100644
index 15f70819f6ce..000000000000
--- a/runners/spark/README.md
+++ /dev/null
@@ -1,124 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-Spark Beam Runner (Spark-Runner)
-================================
-
-## Intro
-
-The Spark-Runner allows users to execute data pipelines written against the Apache Beam API
-with Apache Spark. This runner allows to execute both batch and streaming pipelines on top of the Spark engine.
-
-## Overview
-
-### Features
-
-- ParDo
-- GroupByKey
-- Combine
-- Windowing
-- Flatten
-- View
-- Side inputs/outputs
-- Encoding
-
-### Fault-Tolerance
-
-The Spark runner fault-tolerance guarantees the same guarantees as [Apache Spark](http://spark.apache.org/).
-
-### Monitoring
-
-The Spark runner supports user-defined counters via Beam Aggregators implemented on top of Spark's [Accumulators](http://spark.apache.org/docs/1.6.3/programming-guide.html#accumulators).  
-The Aggregators (defined by the pipeline author) and Spark's internal metrics are reported using Spark's [metrics system](http://spark.apache.org/docs/1.6.3/monitoring.html#metrics).  
-Spark also provides a web UI for monitoring, more details [here](http://spark.apache.org/docs/1.6.3/monitoring.html).
-
-## Beam Model support
-
-### Batch
-
-The Spark runner provides full support for the Beam Model in batch processing via Spark [RDD](http://spark.apache.org/docs/1.6.3/programming-guide.html#resilient-distributed-datasets-rdds)s.
-
-### Streaming
-
-Providing full support for the Beam Model in streaming pipelines is under development. To follow-up you can subscribe to our [mailing list](http://beam.apache.org/get-started/support/).
-
-### issue tracking
-
-See [Beam JIRA](https://issues.apache.org/jira/browse/BEAM) (runner-spark)
-
-
-## Getting Started
-
-To get the latest version of the Spark Runner, first clone the Beam repository:
-
-    git clone https://github.com/apache/beam
-
-    
-Then switch to the newly created directory and run Maven to build the Apache Beam:
-
-    cd beam
-    mvn clean install -DskipTests
-
-Now Apache Beam and the Spark Runner are installed in your local maven repository.
-
-If we wanted to run a Beam pipeline with the default options of a Spark instance in local mode, 
-we would do the following:
-
-    Pipeline p = <logic for pipeline creation >
-    PipelineResult result = p.run();
-    result.waitUntilFinish();
-
-To create a pipeline runner to run against a different Spark cluster, with a custom master url we
-would do the following:
-
-    SparkPipelineOptions options = PipelineOptionsFactory.as(SparkPipelineOptions.class);
-    options.setSparkMaster("spark://host:port");
-    Pipeline p = <logic for pipeline creation >
-    PipelineResult result = p.run();
-    result.waitUntilFinish();
-
-## Word Count Example
-
-First download a text document to use as input:
-
-    curl http://www.gutenberg.org/cache/epub/1128/pg1128.txt > /tmp/kinglear.txt
-    
-Switch to the Spark runner directory:
-
-    cd runners/spark
-    
-Then run the [word count example][wc] from the SDK using a Spark instance in local mode:
-
-    mvn exec:exec -DmainClass=org.apache.beam.runners.spark.examples.WordCount \
-          -Dinput=/tmp/kinglear.txt -Doutput=/tmp/out -Drunner=SparkRunner \
-          -DsparkMaster=local
-
-Check the output by running:
-
-    head /tmp/out-00000-of-00001
-
-__Note: running examples using `mvn exec:exec` only works for Spark local mode at the
-moment. See the next section for how to run on a cluster.__
-
-[wc]: https://github.com/apache/beam/blob/master/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
-## Running on a Cluster
-
-Spark Beam pipelines can be run on a cluster using the `spark-submit` command.
-
-TBD pending native HDFS support (currently blocked by [BEAM-59](https://issues.apache.org/jira/browse/BEAM-59)).
diff --git a/sdks/java/javadoc/README.md b/sdks/java/javadoc/README.md
deleted file mode 100644
index bb17c3f331f3..000000000000
--- a/sdks/java/javadoc/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-# SDK Javadoc
-
-This directory contains package-info files for external javadoc we would like
-our javadoc to link to using `-linkoffline`.
diff --git a/sdks/python/apache_beam/tests/data/README.md b/sdks/python/apache_beam/tests/data/README.md
deleted file mode 100644
index 58563423ac40..000000000000
--- a/sdks/python/apache_beam/tests/data/README.md
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-# Generating Test Data Files
-* [privatekey.p12](https://github.com/google/oauth2client/blob/master/tests/data/privatekey.p12)

From 78c29238aa471138fb55681d8700b095a3606e0d Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 14:54:17 -0700
Subject: [PATCH 120/387] [BEAM-2224] Fix temp file management on Windows.

---
 .../portability/maptask_executor_runner_test.py        | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
index b52c73c3948e..aebd2e1356db 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
@@ -16,6 +16,7 @@
 #
 
 import logging
+import os
 import tempfile
 import unittest
 
@@ -181,12 +182,17 @@ def test_combine_per_key(self):
       assert_that(res, equal_to([('a', 1.5), ('b', 3.0)]))
 
   def test_read(self):
-    with tempfile.NamedTemporaryFile() as temp_file:
+    # Can't use NamedTemporaryFile as a context
+    # due to https://bugs.python.org/issue14243
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+    try:
       temp_file.write('a\nb\nc')
-      temp_file.flush()
+      temp_file.close()
       with self.create_pipeline() as p:
         assert_that(p | beam.io.ReadFromText(temp_file.name),
                     equal_to(['a', 'b', 'c']))
+    finally:
+      os.unlink(temp_file.name)
 
   def test_windowing(self):
     with self.create_pipeline() as p:

From fd1dda09fc4db48cfbc21afd6310320533fffe78 Mon Sep 17 00:00:00 2001
From: Reuven Lax <relax@google.com>
Date: Sat, 29 Apr 2017 07:33:54 -0700
Subject: [PATCH 121/387] [BEAM-2154] More scalable dynamic BigQueryIO.Write

If too many tables are generated in a bundle, spill and group the results
before writing files. Generating hundreds or thousands of file write buffers
in a single bundle was causing workers to crash with out of memory.
---
 .../beam/sdk/io/gcp/bigquery/BatchLoads.java  | 122 +++++++++++++++---
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java  |  35 ++---
 .../sdk/io/gcp/bigquery/ShardedKeyCoder.java  |   3 +-
 .../sdk/io/gcp/bigquery/TableRowWriter.java   |  59 ++++-----
 .../io/gcp/bigquery/WriteBundlesToFiles.java  | 111 +++++++++++++---
 .../bigquery/WriteGroupedRecordsToFiles.java  |  68 ++++++++++
 .../sdk/io/gcp/bigquery/WritePartition.java   |  38 +++---
 .../beam/sdk/io/gcp/bigquery/WriteRename.java |   9 +-
 .../beam/sdk/io/gcp/bigquery/WriteResult.java |   2 +-
 .../beam/sdk/io/gcp/bigquery/WriteTables.java |   7 +-
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java   |  81 ++++++++----
 .../sdk/io/gcp/bigquery/FakeJobService.java   |   6 +-
 12 files changed, 396 insertions(+), 145 deletions(-)
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
index ba64ab108656..0abd46986f5a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java
@@ -22,6 +22,7 @@
 import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation;
 
 import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import java.util.List;
@@ -32,11 +33,15 @@
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.WriteBundlesToFiles.Result;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -49,6 +54,7 @@
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
@@ -57,6 +63,36 @@
 /** PTransform that uses BigQuery batch-load jobs to write a PCollection to BigQuery. */
 class BatchLoads<DestinationT>
     extends PTransform<PCollection<KV<DestinationT, TableRow>>, WriteResult> {
+  // The maximum number of file writers to keep open in a single bundle at a time, since file
+  // writers default to 64mb buffers. This comes into play when writing dynamic table destinations.
+  // The first 20 tables from a single BatchLoads transform will write files inline in the
+  // transform. Anything beyond that might be shuffled.  Users using this transform directly who
+  // know that they are running on workers with sufficient memory can increase this by calling
+  // BatchLoads#setMaxNumWritersPerBundle. This allows the workers to do more work in memory, and
+  // save on the cost of shuffling some of this data.
+  // Keep in mind that specific runners may decide to run multiple bundles in parallel, based on
+  // their own policy.
+  @VisibleForTesting
+  static final int DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE = 20;
+
+  @VisibleForTesting
+  // Maximum number of files in a single partition.
+  static final int MAX_NUM_FILES = 10000;
+
+  @VisibleForTesting
+  // Maximum number of bytes in a single partition -- 11 TiB just under BQ's 12 TiB limit.
+  static final long MAX_SIZE_BYTES = 11 * (1L << 40);
+
+  // The maximum size of a single file - 4TiB, just under the 5 TiB limit.
+  static final long DEFAULT_MAX_FILE_SIZE = 4 * (1L << 40);
+
+  // The maximum number of retries to poll the status of a job.
+  // It sets to {@code Integer.MAX_VALUE} to block until the BigQuery job finishes.
+  static final int LOAD_JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
+
+  // The maximum number of retry jobs.
+  static final int MAX_RETRY_JOBS = 3;
+
   private BigQueryServices bigQueryServices;
   private final WriteDisposition writeDisposition;
   private final CreateDisposition createDisposition;
@@ -65,6 +101,8 @@ class BatchLoads<DestinationT>
   private final boolean singletonTable;
   private final DynamicDestinations<?, DestinationT> dynamicDestinations;
   private final Coder<DestinationT> destinationCoder;
+  private int maxNumWritersPerBundle;
+  private long maxFileSize;
 
   BatchLoads(WriteDisposition writeDisposition, CreateDisposition createDisposition,
              boolean singletonTable,
@@ -76,12 +114,29 @@ class BatchLoads<DestinationT>
     this.singletonTable = singletonTable;
     this.dynamicDestinations = dynamicDestinations;
     this.destinationCoder = destinationCoder;
+    this.maxNumWritersPerBundle = DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE;
+    this.maxFileSize = DEFAULT_MAX_FILE_SIZE;
   }
 
   void setTestServices(BigQueryServices bigQueryServices) {
     this.bigQueryServices = bigQueryServices;
   }
 
+  /** Get the maximum number of file writers that will be open simultaneously in a bundle. */
+  public int getMaxNumWritersPerBundle() {
+    return maxNumWritersPerBundle;
+  }
+
+  /** Set the maximum number of file writers that will be open simultaneously in a bundle. */
+  public void setMaxNumWritersPerBundle(int maxNumWritersPerBundle) {
+    this.maxNumWritersPerBundle = maxNumWritersPerBundle;
+  }
+
+  @VisibleForTesting
+  void setMaxFileSize(long maxFileSize) {
+    this.maxFileSize = maxFileSize;
+  }
+
   @Override
   public void validate(PipelineOptions options) {
     // We will use a BigQuery load job -- validate the temp location.
@@ -107,21 +162,25 @@ public WriteResult expand(PCollection<KV<DestinationT, TableRow>> input) {
     Pipeline p = input.getPipeline();
     final String stepUuid = BigQueryHelpers.randomUUIDString();
 
+    PCollectionView<String> tempFilePrefix =
+        p.apply("Create", Create.of((Void) null))
+            .apply(
+                "GetTempFilePrefix",
+                ParDo.of(
+                    new DoFn<Void, String>() {
+                      @ProcessElement
+                      public void getTempFilePrefix(ProcessContext c) {
+                        c.output(
+                            resolveTempLocation(
+                                c.getPipelineOptions().getTempLocation(),
+                                "BigQueryWriteTemp",
+                                stepUuid));
+                      }
+                    }))
+            .apply("TempFilePrefixView", View.<String>asSingleton());
+
     // Create a singleton job ID token at execution time. This will be used as the base for all
     // load jobs issued from this instance of the transform.
-    PCollection<String> singleton = p
-        .apply("Create", Create.of((Void) null))
-        .apply("GetTempFilePrefix", ParDo.of(new DoFn<Void, String>() {
-          @ProcessElement
-          public void getTempFilePrefix(ProcessContext c) {
-            c.output(
-                resolveTempLocation(
-                    c.getPipelineOptions().getTempLocation(),
-                    "BigQueryWriteTemp",
-                    stepUuid));
-          }
-        }));
-
     PCollectionView<String> jobIdTokenView =
         p.apply("TriggerIdCreation", Create.of("ignored"))
             .apply(
@@ -144,12 +203,37 @@ public String apply(String input) {
     PCollectionView<Map<DestinationT, String>> schemasView =
         inputInGlobalWindow.apply(new CalculateSchemas<>(dynamicDestinations));
 
+    TupleTag<WriteBundlesToFiles.Result<DestinationT>> writtenFilesTag =
+        new TupleTag<WriteBundlesToFiles.Result<DestinationT>>("writtenFiles"){};
+    TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittedRecordsTag =
+        new TupleTag<KV<ShardedKey<DestinationT>, TableRow>>("unwrittenRecords") {};
+    PCollectionTuple writeBundlesTuple = inputInGlobalWindow
+            .apply("WriteBundlesToFiles",
+                ParDo.of(new WriteBundlesToFiles<>(stepUuid, unwrittedRecordsTag,
+                    maxNumWritersPerBundle, maxFileSize))
+                .withOutputTags(writtenFilesTag, TupleTagList.of(unwrittedRecordsTag)));
+    PCollection<WriteBundlesToFiles.Result<DestinationT>> writtenFiles =
+        writeBundlesTuple.get(writtenFilesTag)
+        .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
+
+    // If the bundles contain too many output tables to be written inline to files (due to memory
+    // limits), any unwritten records will be spilled to the unwrittenRecordsTag PCollection.
+    // Group these records by key, and write the files after grouping. Since the record is grouped
+    // by key, we can ensure that only one file is open at a time in each bundle.
+    PCollection<WriteBundlesToFiles.Result<DestinationT>> writtenFilesGrouped =
+        writeBundlesTuple
+            .get(unwrittedRecordsTag)
+            .setCoder(KvCoder.of(ShardedKeyCoder.of(destinationCoder), TableRowJsonCoder.of()))
+            .apply(GroupByKey.<ShardedKey<DestinationT>, TableRow>create())
+            .apply(
+                ParDo.of(new WriteGroupedRecordsToFiles<DestinationT>(tempFilePrefix, maxFileSize))
+                    .withSideInputs(tempFilePrefix))
+            .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
+
     // PCollection of filename, file byte size, and table destination.
     PCollection<WriteBundlesToFiles.Result<DestinationT>> results =
-        inputInGlobalWindow
-            .apply("WriteBundlesToFiles", ParDo.of(
-                new WriteBundlesToFiles<DestinationT>(stepUuid)))
-            .setCoder(WriteBundlesToFiles.ResultCoder.of(destinationCoder));
+        PCollectionList.of(writtenFiles).and(writtenFilesGrouped)
+        .apply(Flatten.<Result<DestinationT>>pCollections());
 
     TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag =
         new TupleTag<KV<ShardedKey<DestinationT>, List<String>>>("multiPartitionsTag") {};
@@ -164,16 +248,18 @@ public String apply(String input) {
     // This transform will look at the set of files written for each table, and if any table has
     // too many files or bytes, will partition that table's files into multiple partitions for
     // loading.
+    PCollection<Void> singleton = p.apply(Create.of((Void) null).withCoder(VoidCoder.of()));
     PCollectionTuple partitions =
         singleton.apply(
             "WritePartition",
             ParDo.of(
                     new WritePartition<>(
                         singletonTable,
+                        tempFilePrefix,
                         resultsView,
                         multiPartitionsTag,
                         singlePartitionTag))
-                .withSideInputs(resultsView)
+                .withSideInputs(tempFilePrefix, resultsView)
                 .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
 
     List<PCollectionView<?>> writeTablesSideInputs =
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 8fb05ffa8c16..cf258ca09f6d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -658,21 +658,6 @@ public static Write<TableRow> writeTableRows() {
   /** Implementation of {@link #write}. */
   @AutoValue
   public abstract static class Write<T> extends PTransform<PCollection<T>, WriteResult> {
-    @VisibleForTesting
-    // Maximum number of files in a single partition.
-    static final int MAX_NUM_FILES = 10000;
-
-    @VisibleForTesting
-    // Maximum number of bytes in a single partition -- 11 TiB just under BQ's 12 TiB limit.
-    static final long MAX_SIZE_BYTES = 11 * (1L << 40);
-
-    // The maximum number of retry jobs.
-    static final int MAX_RETRY_JOBS = 3;
-
-    // The maximum number of retries to poll the status of a job.
-    // It sets to {@code Integer.MAX_VALUE} to block until the BigQuery job finishes.
-    static final int LOAD_JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
-
     @Nullable abstract ValueProvider<String> getJsonTableRef();
     @Nullable abstract SerializableFunction<ValueInSingleWindow<T>, TableDestination>
       getTableFunction();
@@ -687,6 +672,8 @@ public abstract static class Write<T> extends PTransform<PCollection<T>, WriteRe
     /** An option to indicate if table validation is desired. Default is true. */
     abstract boolean getValidate();
     abstract BigQueryServices getBigQueryServices();
+    @Nullable abstract Integer getMaxFilesPerBundle();
+    @Nullable abstract Long getMaxFileSize();
 
     abstract Builder<T> toBuilder();
 
@@ -704,6 +691,8 @@ abstract Builder<T> setTableFunction(
       abstract Builder<T> setTableDescription(String tableDescription);
       abstract Builder<T> setValidate(boolean validate);
       abstract Builder<T> setBigQueryServices(BigQueryServices bigQueryServices);
+      abstract Builder<T> setMaxFilesPerBundle(Integer maxFilesPerBundle);
+      abstract Builder<T> setMaxFileSize(Long maxFileSize);
 
       abstract Write<T> build();
     }
@@ -882,6 +871,16 @@ Write<T> withTestServices(BigQueryServices testServices) {
       return toBuilder().setBigQueryServices(testServices).build();
     }
 
+    @VisibleForTesting
+    Write<T> withMaxFilesPerBundle(int maxFilesPerBundle) {
+      return toBuilder().setMaxFilesPerBundle(maxFilesPerBundle).build();
+    }
+
+    @VisibleForTesting
+    Write<T> withMaxFileSize(long maxFileSize) {
+      return toBuilder().setMaxFileSize(maxFileSize).build();
+    }
+
     @Override
     public void validate(PipelineOptions pipelineOptions) {
       BigQueryOptions options = pipelineOptions.as(BigQueryOptions.class);
@@ -993,6 +992,12 @@ private <DestinationT> WriteResult expandTyped(
             dynamicDestinations,
             destinationCoder);
         batchLoads.setTestServices(getBigQueryServices());
+        if (getMaxFilesPerBundle() != null) {
+          batchLoads.setMaxNumWritersPerBundle(getMaxFilesPerBundle());
+        }
+        if (getMaxFileSize() != null) {
+          batchLoads.setMaxFileSize(getMaxFileSize());
+        }
         return rowsWithDestination.apply(batchLoads);
       }
     }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
index f04e9b9ae7f2..7aefcfa606c0 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
@@ -63,8 +63,7 @@ public void encode(ShardedKey<KeyT> key, OutputStream outStream, Context context
   public ShardedKey<KeyT> decode(InputStream inStream, Context context)
       throws IOException {
     return new ShardedKey<>(
-        keyCoder.decode(inStream, context.nested()),
-        shardNumberCoder.decode(inStream, context));
+        keyCoder.decode(inStream, context.nested()), shardNumberCoder.decode(inStream, context));
   }
 
   @Override
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
index f9d87857be1c..e2db87183272 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
@@ -18,12 +18,15 @@
 
 package org.apache.beam.sdk.io.gcp.bigquery;
 
+import static com.google.common.base.Preconditions.checkState;
+
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.common.io.CountingOutputStream;
 import java.io.IOException;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
+import java.util.UUID;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.io.FileSystems;
@@ -32,20 +35,19 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/** Writes {@TableRow} objects out to a file. Used when doing batch load jobs into BigQuery. */
-class TableRowWriter {
-  private static final Logger LOG = LoggerFactory.getLogger(BigQueryIO.class);
+/** Writes {@link TableRow} objects out to a file. Used when doing batch load jobs into BigQuery. */
+class TableRowWriter implements AutoCloseable {
+  private static final Logger LOG = LoggerFactory.getLogger(TableRowWriter.class);
 
   private static final Coder<TableRow> CODER = TableRowJsonCoder.of();
   private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
-  private final String tempFilePrefix;
-  private String id;
   private ResourceId resourceId;
   private WritableByteChannel channel;
-  protected String mimeType = MimeTypes.TEXT;
   private CountingOutputStream out;
 
-  public static final class Result {
+  private boolean isClosed = false;
+
+  static final class Result {
     final ResourceId resourceId;
     final long byteSize;
 
@@ -55,37 +57,32 @@ public Result(ResourceId resourceId, long byteSize) {
     }
   }
 
-  TableRowWriter(String basename) {
-    this.tempFilePrefix = basename;
-  }
-
-  public final void open(String uId) throws Exception {
-    id = uId;
-    resourceId = FileSystems.matchNewResource(tempFilePrefix + id, false);
-    LOG.debug("Opening {}.", resourceId);
-    channel = FileSystems.create(resourceId, mimeType);
-    try {
-      out = new CountingOutputStream(Channels.newOutputStream(channel));
-      LOG.debug("Writing header to {}.", resourceId);
-    } catch (Exception e) {
-      try {
-        LOG.error("Writing header to {} failed, closing channel.", resourceId);
-        channel.close();
-      } catch (IOException closeException) {
-        LOG.error("Closing channel for {} failed", resourceId);
-      }
-      throw e;
-    }
-    LOG.debug("Starting write of bundle {} to {}.", this.id, resourceId);
+  TableRowWriter(String basename) throws Exception {
+    String uId = UUID.randomUUID().toString();
+    resourceId = FileSystems.matchNewResource(basename + uId, false);
+    LOG.info("Opening TableRowWriter to {}.", resourceId);
+    channel = FileSystems.create(resourceId, MimeTypes.TEXT);
+    out = new CountingOutputStream(Channels.newOutputStream(channel));
   }
 
-  public void write(TableRow value) throws Exception {
+  void write(TableRow value) throws Exception {
     CODER.encode(value, out, Context.OUTER);
     out.write(NEWLINE);
   }
 
-  public final Result close() throws IOException {
+  long getByteSize() {
+    return out.getCount();
+  }
+
+  @Override
+  public void close() throws IOException {
+    checkState(!isClosed, "Already closed");
+    isClosed = true;
     channel.close();
+  }
+
+  Result getResult() {
+    checkState(isClosed, "Not yet closed");
     return new Result(resourceId, out.getCount());
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
index b8960835e440..890979ba2c8d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
@@ -21,6 +21,7 @@
 import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation;
 
 import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.io.InputStream;
@@ -29,31 +30,43 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.io.gcp.bigquery.WriteBundlesToFiles.Result;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Writes each bundle of {@link TableRow} elements out to a separate file using {@link
- * TableRowWriter}.
+ * Writes each bundle of {@link TableRow} elements out to separate file using {@link
+ * TableRowWriter}. Elements destined to different destinations are written to separate files.
+ * The transform will not write an element to a file if it is already writing to
+ * {@link #maxNumWritersPerBundle} files and the element is destined to a new destination. In this
+ * case, the element will be spilled into the output, and the {@link WriteGroupedRecordsToFiles}
+ * transform will take care of writing it to a file.
  */
 class WriteBundlesToFiles<DestinationT>
-    extends DoFn<KV<DestinationT, TableRow>, WriteBundlesToFiles.Result<DestinationT>> {
+    extends DoFn<KV<DestinationT, TableRow>, Result<DestinationT>> {
   private static final Logger LOG = LoggerFactory.getLogger(WriteBundlesToFiles.class);
 
+  // When we spill records, shard the output keys to prevent hotspots. Experiments running up to
+  // 10TB of data have shown a sharding of 10 to be a good choice.
+  private static final int SPILLED_RECORD_SHARDING_FACTOR = 10;
+
   // Map from tablespec to a writer for that table.
   private transient Map<DestinationT, TableRowWriter> writers;
   private transient Map<DestinationT, BoundedWindow> writerWindows;
   private final String stepUuid;
-
+  private final TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittedRecordsTag;
+  private int maxNumWritersPerBundle;
+  private long maxFileSize;
 
   /**
    * The result of the {@link WriteBundlesToFiles} transform. Corresponds to a single output file,
@@ -115,30 +128,66 @@ public List<? extends Coder<?>> getCoderArguments() {
     public void verifyDeterministic() {}
   }
 
-  WriteBundlesToFiles(String stepUuid) {
+  WriteBundlesToFiles(
+      String stepUuid,
+      TupleTag<KV<ShardedKey<DestinationT>, TableRow>> unwrittedRecordsTag,
+      int maxNumWritersPerBundle,
+      long maxFileSize) {
     this.stepUuid = stepUuid;
+    this.unwrittedRecordsTag = unwrittedRecordsTag;
+    this.maxNumWritersPerBundle = maxNumWritersPerBundle;
+    this.maxFileSize = maxFileSize;
   }
 
   @StartBundle
   public void startBundle() {
-    // This must be done each bundle, as by default the {@link DoFn} might be reused between
+    // This must be done for each bundle, as by default the {@link DoFn} might be reused between
     // bundles.
     this.writers = Maps.newHashMap();
     this.writerWindows = Maps.newHashMap();
   }
 
+  TableRowWriter createAndInsertWriter(DestinationT destination, String tempFilePrefix,
+                                       BoundedWindow window) throws Exception {
+    TableRowWriter writer = new TableRowWriter(tempFilePrefix);
+    writers.put(destination, writer);
+    writerWindows.put(destination, window);
+    return writer;
+  }
+
   @ProcessElement
   public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
     String tempFilePrefix = resolveTempLocation(
         c.getPipelineOptions().getTempLocation(), "BigQueryWriteTemp", stepUuid);
-    TableRowWriter writer = writers.get(c.element().getKey());
-    if (writer == null) {
-      writer = new TableRowWriter(tempFilePrefix);
-      writer.open(UUID.randomUUID().toString());
-      writers.put(c.element().getKey(), writer);
-      writerWindows.put(c.element().getKey(), window);
-      LOG.debug("Done opening writer {}", writer);
+    DestinationT destination = c.element().getKey();
+
+    TableRowWriter writer;
+    if (writers.containsKey(destination)) {
+      writer = writers.get(destination);
+    } else {
+      // Only create a new writer if we have fewer than maxNumWritersPerBundle already in this
+      // bundle.
+      if (writers.size() <= maxNumWritersPerBundle) {
+        writer = createAndInsertWriter(destination, tempFilePrefix, window);
+      } else {
+        // This means that we already had too many writers open in this bundle. "spill" this record
+        // into the output. It will be grouped and written to a file in a subsequent stage.
+        c.output(unwrittedRecordsTag,
+            KV.of(ShardedKey.of(destination,
+                ThreadLocalRandom.current().nextInt(SPILLED_RECORD_SHARDING_FACTOR)),
+                c.element().getValue()));
+        return;
+      }
     }
+
+    if (writer.getByteSize() > maxFileSize) {
+      // File is too big. Close it and open a new file.
+      writer.close();
+      TableRowWriter.Result result = writer.getResult();
+      c.output(new Result<>(result.resourceId.toString(), result.byteSize, destination));
+      writer = createAndInsertWriter(destination, tempFilePrefix, window);
+    }
+
     try {
       writer.write(c.element().getValue());
     } catch (Exception e) {
@@ -156,14 +205,36 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
 
   @FinishBundle
   public void finishBundle(FinishBundleContext c) throws Exception {
+    List<Exception> exceptionList = Lists.newArrayList();
+    for (TableRowWriter writer : writers.values()) {
+      try {
+        writer.close();
+      } catch (Exception e) {
+        exceptionList.add(e);
+      }
+    }
+    if (!exceptionList.isEmpty()) {
+      Exception e = new IOException("Failed to close some writers");
+      for (Exception thrown : exceptionList) {
+        e.addSuppressed(thrown);
+      }
+      throw e;
+    }
+
     for (Map.Entry<DestinationT, TableRowWriter> entry : writers.entrySet()) {
-      TableRowWriter.Result result = entry.getValue().close();
-      c.output(
-          new Result<>(result.resourceId.toString(), result.byteSize, entry.getKey()),
-          writerWindows.get(entry.getKey()).maxTimestamp(),
-          writerWindows.get(entry.getKey()));
+      try {
+        DestinationT destination = entry.getKey();
+        TableRowWriter writer = entry.getValue();
+        TableRowWriter.Result result = writer.getResult();
+        c.output(
+            new Result<>(result.resourceId.toString(), result.byteSize, destination),
+            writerWindows.get(destination).maxTimestamp(),
+            writerWindows.get(destination));
+      } catch (Exception e) {
+        exceptionList.add(e);
+      }
     }
     writers.clear();
-    writerWindows.clear();
+
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java
new file mode 100644
index 000000000000..45dc2a83898b
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteGroupedRecordsToFiles.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableRow;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Receives elements grouped by their (sharded) destination, and writes them out to a file.
+ * Since all the elements in the {@link Iterable} are destined to the same table, they are all
+ * written to the same file. Ensures that only one {@link TableRowWriter} is active per bundle.
+ */
+class WriteGroupedRecordsToFiles<DestinationT>
+    extends DoFn<KV<ShardedKey<DestinationT>, Iterable<TableRow>>,
+    WriteBundlesToFiles.Result<DestinationT>> {
+  private static final Logger LOG = LoggerFactory.getLogger(WriteGroupedRecordsToFiles.class);
+
+  private final PCollectionView<String> tempFilePrefix;
+  private final long maxFileSize;
+
+  WriteGroupedRecordsToFiles(PCollectionView<String> tempFilePrefix, long maxFileSize) {
+    this.tempFilePrefix = tempFilePrefix;
+    this.maxFileSize = maxFileSize;
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) throws Exception {
+    String tempFilePrefix = c.sideInput(this.tempFilePrefix);
+    TableRowWriter writer = new TableRowWriter(tempFilePrefix);
+    try (TableRowWriter ignored = writer) {
+      for (TableRow tableRow : c.element().getValue()) {
+        if (writer.getByteSize() > maxFileSize) {
+          writer.close();
+          TableRowWriter.Result result = writer.getResult();
+          c.output(new WriteBundlesToFiles.Result<>(
+              result.resourceId.toString(), result.byteSize, c.element().getKey().getKey()));
+          writer = new TableRowWriter(tempFilePrefix);
+        }
+        writer.write(tableRow);
+      }
+    }
+    TableRowWriter.Result result = writer.getResult();
+    c.output(
+        new WriteBundlesToFiles.Result<>(
+            result.resourceId.toString(), result.byteSize, c.element().getKey().getKey()));
+  }
+
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
index 66004b22b6a5..24693da03236 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
@@ -22,8 +22,6 @@
 import com.google.common.collect.Maps;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.WriteBundlesToFiles.Result;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.values.KV;
@@ -35,9 +33,10 @@
  * tablespec and the list of files corresponding to each partition of that table.
  */
 class WritePartition<DestinationT>
-    extends DoFn<String, KV<ShardedKey<DestinationT>, List<String>>> {
+    extends DoFn<Void, KV<ShardedKey<DestinationT>, List<String>>> {
   private final boolean singletonTable;
-  private final PCollectionView<Iterable<WriteBundlesToFiles.Result<DestinationT>>> resultsView;
+  private final PCollectionView<String> tempFilePrefix;
+  private final PCollectionView<Iterable<WriteBundlesToFiles.Result<DestinationT>>> results;
   private TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag;
   private TupleTag<KV<ShardedKey<DestinationT>, List<String>>> singlePartitionTag;
 
@@ -73,8 +72,8 @@ void addFilename(String filename) {
     // Check to see whether we can add to this partition without exceeding the maximum partition
     // size.
     boolean canAccept(int numFiles, long numBytes) {
-      return this.numFiles + numFiles <= Write.MAX_NUM_FILES
-          && this.byteSize + numBytes <= Write.MAX_SIZE_BYTES;
+      return this.numFiles + numFiles <= BatchLoads.MAX_NUM_FILES
+          && this.byteSize + numBytes <= BatchLoads.MAX_SIZE_BYTES;
     }
   }
 
@@ -101,11 +100,13 @@ void addPartition(PartitionData partition) {
 
   WritePartition(
       boolean singletonTable,
-      PCollectionView<Iterable<WriteBundlesToFiles.Result<DestinationT>>> resultsView,
+      PCollectionView<String> tempFilePrefix,
+      PCollectionView<Iterable<WriteBundlesToFiles.Result<DestinationT>>> results,
       TupleTag<KV<ShardedKey<DestinationT>, List<String>>> multiPartitionsTag,
       TupleTag<KV<ShardedKey<DestinationT>, List<String>>> singlePartitionTag) {
     this.singletonTable = singletonTable;
-    this.resultsView = resultsView;
+    this.results = results;
+    this.tempFilePrefix = tempFilePrefix;
     this.multiPartitionsTag = multiPartitionsTag;
     this.singlePartitionTag = singlePartitionTag;
   }
@@ -113,21 +114,20 @@ void addPartition(PartitionData partition) {
   @ProcessElement
   public void processElement(ProcessContext c) throws Exception {
     List<WriteBundlesToFiles.Result<DestinationT>> results =
-        Lists.newArrayList(c.sideInput(resultsView));
+        Lists.newArrayList(c.sideInput(this.results));
 
     // If there are no elements to write _and_ the user specified a constant output table, then
     // generate an empty table of that name.
     if (results.isEmpty() && singletonTable) {
-        TableRowWriter writer = new TableRowWriter(c.element());
-        writer.open(UUID.randomUUID().toString());
-        TableRowWriter.Result writerResult = writer.close();
-        // Return a null destination in this case - the constant DynamicDestinations class will
-        // resolve it to the singleton output table.
-        results.add(
-            new Result<DestinationT>(
-                writerResult.resourceId.toString(),
-                writerResult.byteSize,
-                null));
+      String tempFilePrefix = c.sideInput(this.tempFilePrefix);
+      TableRowWriter writer = new TableRowWriter(tempFilePrefix);
+      writer.close();
+      TableRowWriter.Result writerResult = writer.getResult();
+      // Return a null destination in this case - the constant DynamicDestinations class will
+      // resolve it to the singleton output table.
+      results.add(
+          new Result<DestinationT>(
+              writerResult.resourceId.toString(), writerResult.byteSize, null));
     }
 
     Map<DestinationT, DestinationData> currentResults = Maps.newHashMap();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
index bf9d9f1f8000..f641b327db78 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
@@ -29,7 +29,6 @@
 import java.util.Map;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
@@ -43,7 +42,7 @@
 /**
  * Copies temporary tables to destination table.
  */
-class WriteRename extends DoFn<String, Void> {
+class WriteRename extends DoFn<Void, Void> {
   private static final Logger LOG = LoggerFactory.getLogger(WriteRename.class);
 
   private final BigQueryServices bqServices;
@@ -123,13 +122,13 @@ private void copy(
 
     String projectId = ref.getProjectId();
     Job lastFailedCopyJob = null;
-    for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
+    for (int i = 0; i < BatchLoads.MAX_RETRY_JOBS; ++i) {
       String jobId = jobIdPrefix + "-" + i;
       JobReference jobRef = new JobReference()
           .setProjectId(projectId)
           .setJobId(jobId);
       jobService.startCopyJob(jobRef, copyConfig);
-      Job copyJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
+      Job copyJob = jobService.pollJob(jobRef, BatchLoads.LOAD_JOB_POLL_MAX_RETRIES);
       Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
       switch (jobStatus) {
         case SUCCEEDED:
@@ -154,7 +153,7 @@ private void copy(
         "Failed to create copy job with id prefix %s, "
             + "reached max retries: %d, last failed copy job: %s.",
         jobIdPrefix,
-        Write.MAX_RETRY_JOBS,
+        BatchLoads.MAX_RETRY_JOBS,
         BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
index bc18e8e032e9..db0be3a96825 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java
@@ -29,7 +29,7 @@
 /**
  * The result of a {@link BigQueryIO.Write} transform.
  */
-final class WriteResult implements POutput {
+public final class WriteResult implements POutput {
 
   private final Pipeline pipeline;
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
index 83ff16bb67d1..c5494d883483 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
@@ -35,7 +35,6 @@
 import org.apache.beam.sdk.io.fs.MoveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
@@ -149,11 +148,11 @@ private void load(
 
     String projectId = ref.getProjectId();
     Job lastFailedLoadJob = null;
-    for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
+    for (int i = 0; i < BatchLoads.MAX_RETRY_JOBS; ++i) {
       String jobId = jobIdPrefix + "-" + i;
       JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId);
       jobService.startLoadJob(jobRef, loadConfig);
-      Job loadJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
+      Job loadJob = jobService.pollJob(jobRef, BatchLoads.LOAD_JOB_POLL_MAX_RETRIES);
       Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
       switch (jobStatus) {
         case SUCCEEDED:
@@ -181,7 +180,7 @@ private void load(
             "Failed to create load job with id prefix %s, "
                 + "reached max retries: %d, last failed load job: %s.",
             jobIdPrefix,
-            Write.MAX_RETRY_JOBS,
+            BatchLoads.MAX_RETRY_JOBS,
             BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 0d3f0003fb1f..d60c7214b701 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -70,6 +70,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.beam.sdk.Pipeline;
@@ -490,9 +491,19 @@ public void writeDynamicDestinations(boolean streaming) throws Exception {
     p.apply("Create SideInput2", Create.of(KV.of("a", "a"), KV.of("b", "b"), KV.of("c", "c")))
         .apply("AsMap", View.<String, String>asMap());
 
-    PCollection<String> users = p.apply("CreateUsers",
-        Create.of("bill1", "sam2", "laurence3")
-            .withCoder(StringUtf8Coder.of()))
+    final List<String> allUsernames = ImmutableList.of("bill", "bob", "randolph");
+    List<String> userList = Lists.newArrayList();
+    // Make sure that we generate enough users so that WriteBundlesToFiles is forced to spill to
+    // WriteGroupedRecordsToFiles.
+    for (int i = 0; i < BatchLoads.DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE * 10; ++i) {
+      // Every user has 10 nicknames.
+      for (int j = 0; j < 1; ++j) {
+        String nickname = allUsernames.get(
+            ThreadLocalRandom.current().nextInt(allUsernames.size()));
+        userList.add(nickname + i);
+      }
+    }
+    PCollection<String> users = p.apply("CreateUsers", Create.of(userList))
         .apply(Window.into(new PartitionedGlobalWindows<>(
             new SerializableFunction<String, String>() {
               @Override
@@ -506,6 +517,8 @@ public String apply(String arg) {
     }
     users.apply("WriteBigQuery", BigQueryIO.<String>write()
             .withTestServices(fakeBqServices)
+            .withMaxFilesPerBundle(5)
+            .withMaxFileSize(10)
             .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
             .withFormatFunction(new SerializableFunction<String, TableRow>() {
               @Override
@@ -567,12 +580,24 @@ private void verifySideInputs() {
     File tempDir = new File(bqOptions.getTempLocation());
     testNumFiles(tempDir, 0);
 
-    assertThat(datasetService.getAllRows("project-id", "dataset-id", "userid-1"),
-        containsInAnyOrder(new TableRow().set("name", "bill").set("id", 1)));
-    assertThat(datasetService.getAllRows("project-id", "dataset-id", "userid-2"),
-        containsInAnyOrder(new TableRow().set("name", "sam").set("id", 2)));
-    assertThat(datasetService.getAllRows("project-id", "dataset-id", "userid-3"),
-        containsInAnyOrder(new TableRow().set("name", "laurence").set("id", 3)));
+    Map<Integer, List<TableRow>> expectedTableRows = Maps.newHashMap();
+    for (int i = 0; i < userList.size(); ++i) {
+      Matcher matcher = userPattern.matcher(userList.get(i));
+      checkState(matcher.matches());
+      String nickname = matcher.group(1);
+      int userid = Integer.valueOf(matcher.group(2));
+      List<TableRow> expected = expectedTableRows.get(userid);
+      if (expected == null) {
+        expected = Lists.newArrayList();
+        expectedTableRows.put(userid, expected);
+      }
+      expected.add(new TableRow().set("name", nickname).set("id", userid));
+    }
+
+    for (Map.Entry<Integer, List<TableRow>> entry : expectedTableRows.entrySet()) {
+      assertThat(datasetService.getAllRows("project-id", "dataset-id", "userid-" + entry.getKey()),
+          containsInAnyOrder(Iterables.toArray(entry.getValue(), TableRow.class)));
+    }
   }
 
   @Test
@@ -1655,7 +1680,7 @@ public void testWritePartitionEmptyData() throws Exception {
 
   @Test
   public void testWritePartitionSinglePartition() throws Exception {
-    long numFiles = BigQueryIO.Write.MAX_NUM_FILES;
+    long numFiles = BatchLoads.MAX_NUM_FILES;
     long fileSize = 1;
 
     // One partition is needed.
@@ -1665,7 +1690,7 @@ public void testWritePartitionSinglePartition() throws Exception {
 
   @Test
   public void testWritePartitionManyFiles() throws Exception {
-    long numFiles = BigQueryIO.Write.MAX_NUM_FILES * 3;
+    long numFiles = BatchLoads.MAX_NUM_FILES * 3;
     long fileSize = 1;
 
     // One partition is needed for each group of BigQueryWrite.MAX_NUM_FILES files.
@@ -1676,7 +1701,7 @@ public void testWritePartitionManyFiles() throws Exception {
   @Test
   public void testWritePartitionLargeFileSize() throws Exception {
     long numFiles = 10;
-    long fileSize = BigQueryIO.Write.MAX_SIZE_BYTES / 3;
+    long fileSize = BatchLoads.MAX_SIZE_BYTES / 3;
 
     // One partition is needed for each group of three files.
     long expectedNumPartitions = 4;
@@ -1726,22 +1751,25 @@ private void testWritePartition(long numTables, long numFilesPerTable, long file
     TupleTag<KV<ShardedKey<String>, List<String>>> singlePartitionTag =
         new TupleTag<KV<ShardedKey<String>, List<String>>>("singlePartitionTag") {};
 
-    PCollection<WriteBundlesToFiles.Result<String>> filesPCollection =
-        p.apply(Create.of(files)
-        .withCoder(WriteBundlesToFiles.ResultCoder.of(StringUtf8Coder.of())));
     PCollectionView<Iterable<WriteBundlesToFiles.Result<String>>> resultsView =
-        PCollectionViews.iterableView(
-        filesPCollection,
-        WindowingStrategy.globalDefault(),
-        WriteBundlesToFiles.ResultCoder.of(StringUtf8Coder.of()));
+        p.apply(
+                Create.of(files)
+                    .withCoder(WriteBundlesToFiles.ResultCoder.of(StringUtf8Coder.of())))
+            .apply(View.<WriteBundlesToFiles.Result<String>>asIterable());
+
+    String tempFilePrefix = testFolder.newFolder("BigQueryIOTest").getAbsolutePath();
+    PCollectionView<String> tempFilePrefixView =
+        p.apply(Create.of(tempFilePrefix)).apply(View.<String>asSingleton());
 
     WritePartition<String> writePartition =
-        new WritePartition<>(isSingleton, resultsView, multiPartitionsTag, singlePartitionTag);
+        new WritePartition<>(
+            isSingleton, tempFilePrefixView, resultsView, multiPartitionsTag, singlePartitionTag);
 
-    DoFnTester<String, KV<ShardedKey<String>, List<String>>> tester =
+    DoFnTester<Void, KV<ShardedKey<String>, List<String>>> tester =
         DoFnTester.of(writePartition);
     tester.setSideInput(resultsView, GlobalWindow.INSTANCE, files);
-    tester.processElement(testFolder.newFolder("BigQueryIOTest").getAbsolutePath());
+    tester.setSideInput(tempFilePrefixView, GlobalWindow.INSTANCE, tempFilePrefix);
+    tester.processElement(null);
 
     List<KV<ShardedKey<String>, List<String>>> partitions;
     if (expectedNumPartitionsPerTable > 1) {
@@ -1897,11 +1925,10 @@ public void testRemoveTemporaryFiles() throws Exception {
     int numFiles = 10;
     List<String> fileNames = Lists.newArrayList();
     String tempFilePrefix = bqOptions.getTempLocation() + "/";
-    TableRowWriter writer = new TableRowWriter(tempFilePrefix);
     for (int i = 0; i < numFiles; ++i) {
-      String fileName = String.format("files%05d", i);
-      writer.open(fileName);
-      fileNames.add(writer.close().resourceId.toString());
+      TableRowWriter writer = new TableRowWriter(tempFilePrefix);
+      writer.close();
+      fileNames.add(writer.getResult().resourceId.toString());
     }
     fileNames.add(tempFilePrefix + String.format("files%05d", numFiles));
 
@@ -1992,7 +2019,7 @@ public void processElement(ProcessContext c) {
         CreateDisposition.CREATE_IF_NEEDED,
         tempTablesView);
 
-    DoFnTester<String, Void> tester = DoFnTester.of(writeRename);
+    DoFnTester<Void, Void> tester = DoFnTester.of(writeRename);
     tester.setSideInput(tempTablesView, GlobalWindow.INSTANCE, tempTables);
     tester.setSideInput(jobIdTokenView, GlobalWindow.INSTANCE, jobIdToken);
     tester.processElement(null);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
index 50be0bb8156d..ed6a0beff703 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
@@ -76,9 +76,9 @@
  */
 class FakeJobService implements JobService, Serializable {
   static final JsonFactory JSON_FACTORY = Transport.getJsonFactory();
-  // Whenever a job is started, the first 5 calls to GetJob will report the job as pending,
-  // the next 5 will return the job as running, and only then will the job report as done.
-  private static final int GET_JOBS_TRANSITION_INTERVAL = 5;
+  // Whenever a job is started, the first 2 calls to GetJob will report the job as pending,
+  // the next 2 will return the job as running, and only then will the job report as done.
+  private static final int GET_JOBS_TRANSITION_INTERVAL = 2;
 
   private FakeDatasetService datasetService;
 

From 87578a6a69f15bca502d9b4c15999aa383e539a5 Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Fri, 5 May 2017 19:33:45 -0700
Subject: [PATCH 122/387] Use guava Base64 encoding instead of google api
 client

---
 .../apache/beam/sdk/coders/StructuralByteArray.java    |  4 ++--
 .../apache/beam/sdk/testing/MatcherDeserializer.java   |  4 ++--
 .../org/apache/beam/sdk/testing/MatcherSerializer.java |  4 ++--
 .../main/java/org/apache/beam/sdk/util/CoderUtils.java | 10 +++++++---
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
index 226f79cd429d..edde68e95bde 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java
@@ -17,8 +17,8 @@
  */
 package org.apache.beam.sdk.coders;
 
-import static com.google.api.client.util.Base64.encodeBase64String;
 
+import com.google.common.io.BaseEncoding;
 import java.util.Arrays;
 
 /**
@@ -53,6 +53,6 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return "base64:" + encodeBase64String(value);
+    return "base64:" + BaseEncoding.base64().encode(value);
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
index 6ca07badac92..e7aa5a7f4283 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java
@@ -22,7 +22,7 @@
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonDeserializer;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.client.util.Base64;
+import com.google.common.io.BaseEncoding;
 import java.io.IOException;
 import org.apache.beam.sdk.util.SerializableUtils;
 
@@ -36,7 +36,7 @@ public SerializableMatcher<?> deserialize(JsonParser jsonParser,
       throws IOException, JsonProcessingException {
     ObjectNode node = jsonParser.readValueAsTree();
     String matcher = node.get("matcher").asText();
-    byte[] in = Base64.decodeBase64(matcher);
+    byte[] in = BaseEncoding.base64().decode(matcher);
     return (SerializableMatcher<?>) SerializableUtils
         .deserializeFromByteArray(in, "SerializableMatcher");
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
index 2b4584c8cdac..35375f66047d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java
@@ -21,7 +21,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.SerializerProvider;
-import com.google.api.client.util.Base64;
+import com.google.common.io.BaseEncoding;
 import java.io.IOException;
 import org.apache.beam.sdk.util.SerializableUtils;
 
@@ -33,7 +33,7 @@ class MatcherSerializer extends JsonSerializer<SerializableMatcher<?>> {
   public void serialize(SerializableMatcher<?> matcher, JsonGenerator jsonGenerator,
       SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
     byte[] out = SerializableUtils.serializeToByteArray(matcher);
-    String encodedString = Base64.encodeBase64String(out);
+    String encodedString = BaseEncoding.base64().encode(out);
     jsonGenerator.writeStartObject();
     jsonGenerator.writeStringField("matcher", encodedString);
     jsonGenerator.writeEndObject();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
index 3380a102e68a..da778296863d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java
@@ -17,8 +17,9 @@
  */
 package org.apache.beam.sdk.util;
 
-import com.google.api.client.util.Base64;
 import com.google.common.base.Throwables;
+import com.google.common.io.BaseEncoding;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -26,6 +27,7 @@
 import java.io.OutputStream;
 import java.lang.ref.SoftReference;
 import java.lang.reflect.ParameterizedType;
+
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.values.TypeDescriptor;
@@ -163,7 +165,7 @@ public static <T> T clone(Coder<T> coder, T value) throws CoderException {
   public static <T> String encodeToBase64(Coder<T> coder, T value)
       throws CoderException {
     byte[] rawValue = encodeToByteArray(coder, value);
-    return Base64.encodeBase64URLSafeString(rawValue);
+    return BaseEncoding.base64Url().omitPadding().encode(rawValue);
   }
 
   /**
@@ -171,7 +173,9 @@ public static <T> String encodeToBase64(Coder<T> coder, T value)
    */
   public static <T> T decodeFromBase64(Coder<T> coder, String encodedValue) throws CoderException {
     return decodeFromSafeStream(
-        coder, new ByteArrayInputStream(Base64.decodeBase64(encodedValue)), Coder.Context.OUTER);
+        coder,
+        new ByteArrayInputStream(BaseEncoding.base64Url().omitPadding().decode(encodedValue)),
+        Coder.Context.OUTER);
   }
 
   /**

From 367fcb28d544934797d25cb34d54136b2d7d6e99 Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Fri, 5 May 2017 19:24:51 -0700
Subject: [PATCH 123/387] Remove google api BackOff usage from sdks/core

---
 .../beam/examples/common/ExampleUtils.java    |  6 +-
 .../beam/examples/WindowedWordCountIT.java    |  2 +-
 runners/google-cloud-dataflow-java/pom.xml    |  2 +-
 .../runners/dataflow/DataflowPipelineJob.java | 18 +++--
 .../runners/dataflow/util/PackageUtil.java    |  3 +-
 .../dataflow/DataflowPipelineJobTest.java     | 13 ++-
 .../dataflow/util/PackageUtilTest.java        |  2 +-
 runners/spark/pom.xml                         |  5 --
 .../runners/spark/io/MicrobatchSource.java    |  2 +-
 sdks/java/core/pom.xml                        |  5 --
 .../io/BoundedReadFromUnboundedSource.java    |  2 +-
 .../beam/sdk/testing/FileChecksumMatcher.java |  2 +-
 .../org/apache/beam/sdk/util/BackOff.java     | 81 +++++++++++++++++++
 .../apache/beam/sdk/util/BackOffUtils.java    | 57 +++++++++++++
 .../beam/sdk/util/ExplicitShardedFile.java    |  3 -
 .../apache/beam/sdk/util/FluentBackoff.java   |  1 -
 .../beam/sdk/util/NumberedShardedFile.java    |  3 -
 .../org/apache/beam/sdk/util/ShardedFile.java |  2 -
 .../org/apache/beam/sdk/util/Sleeper.java     | 48 +++++++++++
 .../apache/beam/SdkCoreApiSurfaceTest.java    |  1 -
 .../beam/sdk/testing/ExpectedLogsTest.java    |  2 +-
 .../sdk/testing/FileChecksumMatcherTest.java  |  5 --
 .../sdk/testing/SystemNanoTimeSleeper.java    |  2 +-
 .../beam/sdk/util/FluentBackoffTest.java      |  1 -
 .../sdk/util/NumberedShardedFileTest.java     | 14 ++--
 .../extensions/gcp/options/GcpOptions.java    |  3 +-
 .../apache/beam/sdk/util/BackOffAdapter.java  | 43 ++++++++++
 .../org/apache/beam/sdk/util/GcsUtil.java     | 13 +--
 .../sdk/util/UploadIdResponseInterceptor.java |  0
 .../sdk/util}/FastNanoClockAndSleeper.java    |  4 +-
 .../util}/FastNanoClockAndSleeperTest.java    |  2 +-
 .../org/apache/beam/sdk/util/GcsUtilTest.java | 23 +++---
 .../util/UploadIdResponseInterceptorTest.java |  0
 .../io/gcp/bigquery/BigQueryServicesImpl.java | 81 ++++++++-----------
 .../bigquery/BigQueryTableRowIterator.java    |  7 +-
 .../sdk/io/gcp/datastore/DatastoreV1.java     |  6 +-
 .../bigquery/BigQueryServicesImplTest.java    | 24 ++++--
 .../sdk/io/gcp/bigquery/FakeJobService.java   | 12 +--
 .../beam/sdk/io/gcp/datastore/V1TestUtil.java |  6 +-
 .../sdk/io/gcp/testing/BigqueryMatcher.java   |  4 +-
 .../io/gcp/testing/BigqueryMatcherTest.java   |  7 +-
 41 files changed, 372 insertions(+), 145 deletions(-)
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOff.java
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOffUtils.java
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/Sleeper.java
 create mode 100644 sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/BackOffAdapter.java
 rename sdks/java/{core => extensions/google-cloud-platform-core}/src/main/java/org/apache/beam/sdk/util/UploadIdResponseInterceptor.java (100%)
 rename sdks/java/{core/src/test/java/org/apache/beam/sdk/testing => extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util}/FastNanoClockAndSleeper.java (94%)
 rename sdks/java/{core/src/test/java/org/apache/beam/sdk/testing => extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util}/FastNanoClockAndSleeperTest.java (98%)
 rename sdks/java/{core => extensions/google-cloud-platform-core}/src/test/java/org/apache/beam/sdk/util/UploadIdResponseInterceptorTest.java (100%)

diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java
index 409085a739a7..6e4698fd04e9 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java
@@ -20,9 +20,6 @@
 import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.client.googleapis.services.AbstractGoogleClientRequest;
 import com.google.api.client.http.HttpRequestInitializer;
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.BackOffUtils;
-import com.google.api.client.util.Sleeper;
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.Bigquery.Datasets;
 import com.google.api.services.bigquery.Bigquery.Tables;
@@ -51,8 +48,11 @@
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.BackOff;
+import org.apache.beam.sdk.util.BackOffUtils;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
+import org.apache.beam.sdk.util.Sleeper;
 import org.apache.beam.sdk.util.Transport;
 import org.joda.time.Duration;
 
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index f7e35c023172..93c4543e0739 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -19,7 +19,6 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-import com.google.api.client.util.Sleeper;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -47,6 +46,7 @@
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.NumberedShardedFile;
 import org.apache.beam.sdk.util.ShardedFile;
+import org.apache.beam.sdk.util.Sleeper;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 import org.joda.time.Duration;
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index b57904122b54..adbd4d70bd25 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170506</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170508</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index 23084ed50498..2d2398314cb8 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -46,6 +46,7 @@
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.metrics.MetricResults;
 import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
@@ -285,9 +286,10 @@ State waitUntilFinish(
 
     BackOff backoff;
     if (!duration.isLongerThan(Duration.ZERO)) {
-      backoff = MESSAGES_BACKOFF_FACTORY.backoff();
+      backoff = BackOffAdapter.toGcpBackOff(MESSAGES_BACKOFF_FACTORY.backoff());
     } else {
-      backoff = MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(duration).backoff();
+      backoff = BackOffAdapter.toGcpBackOff(
+          MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(duration).backoff());
     }
 
     // This function tracks the cumulative time from the *first request* to enforce the wall-clock
@@ -299,7 +301,10 @@ State waitUntilFinish(
     do {
       // Get the state of the job before listing messages. This ensures we always fetch job
       // messages after the job finishes to ensure we have all them.
-      state = getStateWithRetries(STATUS_BACKOFF_FACTORY.withMaxRetries(0).backoff(), sleeper);
+      state = getStateWithRetries(
+          BackOffAdapter.toGcpBackOff(
+              STATUS_BACKOFF_FACTORY.withMaxRetries(0).backoff()),
+          sleeper);
       boolean hasError = state == State.UNKNOWN;
 
       if (messageHandler != null && !hasError) {
@@ -354,7 +359,8 @@ State waitUntilFinish(
           Duration consumed = Duration.millis((nanosConsumed + 999999) / 1000000);
           Duration remaining = duration.minus(consumed);
           if (remaining.isLongerThan(Duration.ZERO)) {
-            backoff = MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(remaining).backoff();
+            backoff = BackOffAdapter.toGcpBackOff(
+                MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(remaining).backoff());
           } else {
             // If there is no time remaining, don't bother backing off.
             backoff = BackOff.STOP_BACKOFF;
@@ -437,7 +443,9 @@ public State getState() {
       return terminalState;
     }
 
-    return getStateWithRetries(STATUS_BACKOFF_FACTORY.backoff(), Sleeper.DEFAULT);
+    return getStateWithRetries(
+        BackOffAdapter.toGcpBackOff(STATUS_BACKOFF_FACTORY.backoff()),
+        Sleeper.DEFAULT);
   }
 
   /**
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java
index 5ddcd29ff163..931f7ea4a0a8 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java
@@ -54,6 +54,7 @@
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.CreateOptions;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
+import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.ZipFiles;
 import org.joda.time.Duration;
@@ -210,7 +211,7 @@ private static void stageOnePackage(
       }
 
       // Upload file, retrying on failure.
-      BackOff backoff = BACKOFF_FACTORY.backoff();
+      BackOff backoff = BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff());
       while (true) {
         try {
           LOG.debug("Uploading classpath element {} to {}", source, target);
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
index df894d2cb33a..3c261d041e07 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
@@ -56,8 +56,9 @@
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.ExpectedLogs;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
 import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.BackOffAdapter;
+import org.apache.beam.sdk.util.FastNanoClockAndSleeper;
 import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
@@ -346,7 +347,10 @@ public void testGetStateReturnsServiceState() throws Exception {
 
     assertEquals(
         State.RUNNING,
-        job.getStateWithRetries(DataflowPipelineJob.STATUS_BACKOFF_FACTORY.backoff(), fastClock));
+        job.getStateWithRetries(
+            BackOffAdapter.toGcpBackOff(
+                DataflowPipelineJob.STATUS_BACKOFF_FACTORY.backoff()),
+            fastClock));
   }
 
   @Test
@@ -367,7 +371,10 @@ public void testGetStateWithExceptionReturnsUnknown() throws Exception {
     long startTime = fastClock.nanoTime();
     assertEquals(
         State.UNKNOWN,
-        job.getStateWithRetries(DataflowPipelineJob.STATUS_BACKOFF_FACTORY.backoff(), fastClock));
+        job.getStateWithRetries(
+            BackOffAdapter.toGcpBackOff(
+                DataflowPipelineJob.STATUS_BACKOFF_FACTORY.backoff()),
+            fastClock));
     long timeDiff = TimeUnit.NANOSECONDS.toMillis(fastClock.nanoTime() - startTime);
     checkValidInterval(DataflowPipelineJob.STATUS_POLLING_INTERVAL,
         DataflowPipelineJob.STATUS_POLLING_RETRIES, timeDiff);
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
index 4ae3a77c42ee..c7a660e688fb 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
@@ -79,8 +79,8 @@
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.ExpectedLogs;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
 import org.apache.beam.sdk.testing.RegexMatcher;
+import org.apache.beam.sdk.util.FastNanoClockAndSleeper;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.GcsUtil.StorageObjectOrIOException;
 import org.apache.beam.sdk.util.MimeTypes;
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 94f163dfdc5a..2c8372b754ef 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -189,11 +189,6 @@
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client</artifactId>
-      <version>${google-clients.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-compress</artifactId>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java
index 53d1ba7a5473..3b48caf3ef85 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java
@@ -18,7 +18,6 @@
 
 package org.apache.beam.runners.spark.io;
 
-import com.google.api.client.util.BackOff;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
@@ -36,6 +35,7 @@
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.BackOff;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index a6b89b8fdfb9..7ed6c8a3a553 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -124,11 +124,6 @@
   </build>
 
   <dependencies>
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
index d9adf9297db5..c882447d7ccc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java
@@ -17,7 +17,6 @@
  */
 package org.apache.beam.sdk.io;
 
-import com.google.api.client.util.BackOff;
 import com.google.auto.value.AutoValue;
 import com.google.common.util.concurrent.Uninterruptibles;
 import java.io.IOException;
@@ -35,6 +34,7 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.BackOff;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.values.PBegin;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
index 82a6b7117636..5ed05259b023 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java
@@ -21,7 +21,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import com.google.api.client.util.Sleeper;
 import com.google.common.base.Strings;
 import com.google.common.hash.HashCode;
 import com.google.common.hash.Hashing;
@@ -34,6 +33,7 @@
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.NumberedShardedFile;
 import org.apache.beam.sdk.util.ShardedFile;
+import org.apache.beam.sdk.util.Sleeper;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 import org.joda.time.Duration;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOff.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOff.java
new file mode 100644
index 000000000000..5bc6027c5d8f
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOff.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import java.io.IOException;
+
+/**
+ * Back-off policy when retrying an operation.
+ *
+ * <p><b>Note</b>: This interface is copied from Google API client library to avoid its dependency.
+ */
+public interface BackOff {
+
+  /** Indicates that no more retries should be made for use in {@link #nextBackOffMillis()}. */
+  long STOP = -1L;
+
+  /** Reset to initial state. */
+  void reset() throws IOException;
+
+  /**
+   * Gets the number of milliseconds to wait before retrying the operation or {@link #STOP} to
+   * indicate that no retries should be made.
+   *
+   * <p>
+   * Example usage:
+   * </p>
+   *
+   * <pre>
+   long backOffMillis = backoff.nextBackOffMillis();
+   if (backOffMillis == Backoff.STOP) {
+   // do not retry operation
+   } else {
+   // sleep for backOffMillis milliseconds and retry operation
+   }
+   * </pre>
+   */
+  long nextBackOffMillis() throws IOException;
+
+  /**
+   * Fixed back-off policy whose back-off time is always zero, meaning that the operation is retried
+   * immediately without waiting.
+   */
+  BackOff ZERO_BACKOFF = new BackOff() {
+
+    public void reset() throws IOException {
+    }
+
+    public long nextBackOffMillis() throws IOException {
+      return 0;
+    }
+  };
+
+  /**
+   * Fixed back-off policy that always returns {@code #STOP} for {@link #nextBackOffMillis()},
+   * meaning that the operation should not be retried.
+   */
+  BackOff STOP_BACKOFF = new BackOff() {
+
+    public void reset() throws IOException {
+    }
+
+    public long nextBackOffMillis() throws IOException {
+      return STOP;
+    }
+  };
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOffUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOffUtils.java
new file mode 100644
index 000000000000..aa7461c2f270
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BackOffUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.util;
+
+import java.io.IOException;
+
+/**
+ * Utilities for {@link BackOff}.
+ *
+ * <p><b>Note</b>: This is copied from Google API client library to avoid its dependency.
+ */
+public final class BackOffUtils {
+
+  /**
+   * Runs the next iteration of the back-off policy, and returns whether to continue to retry the
+   * operation.
+   *
+   * <p>
+   * If {@code true}, it will call {@link Sleeper#sleep(long)} with the specified number of
+   * milliseconds from {@link BackOff#nextBackOffMillis()}.
+   * </p>
+   *
+   * @param sleeper sleeper
+   * @param backOff back-off policy
+   * @return whether to continue to back off; in other words, whether
+   *         {@link BackOff#nextBackOffMillis()} did not return {@link BackOff#STOP}
+   * @throws InterruptedException if any thread has interrupted the current thread
+   */
+  public static boolean next(Sleeper sleeper, BackOff backOff)
+      throws InterruptedException, IOException {
+    long backOffTime = backOff.nextBackOffMillis();
+    if (backOffTime == BackOff.STOP) {
+      return false;
+    }
+    sleeper.sleep(backOffTime);
+    return true;
+  }
+
+  private BackOffUtils() {
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
index 0f184de7f8f4..50e5ed173393 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java
@@ -18,9 +18,6 @@
 
 package org.apache.beam.sdk.util;
 
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.BackOffUtils;
-import com.google.api.client.util.Sleeper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.collect.Lists;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java
index 479d7a81451b..468b742dba18 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java
@@ -19,7 +19,6 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
-import com.google.api.client.util.BackOff;
 import com.google.common.base.MoreObjects;
 import org.joda.time.Duration;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
index e18dd96f0046..8889358688ce 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
@@ -21,9 +21,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.BackOffUtils;
-import com.google.api.client.util.Sleeper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.Iterables;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
index ec9ed641976c..5961c4d7e901 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedFile.java
@@ -18,8 +18,6 @@
 
 package org.apache.beam.sdk.util;
 
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.Sleeper;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.List;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Sleeper.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Sleeper.java
new file mode 100644
index 000000000000..d180ec68bde6
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Sleeper.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+/**
+ * Sleeper interface to use for requesting the current thread to sleep as specified in
+ * {@link Thread#sleep(long)}.
+ *
+ * <p>
+ * The default implementation can be accessed at {@link #DEFAULT}. Primarily used for testing.
+ * </p>
+ *
+ * <p><b>Note</b>: This interface is copied from Google API client library to avoid its dependency.
+ */
+public interface Sleeper {
+
+  /**
+   * Causes the currently executing thread to sleep (temporarily cease execution) for the specified
+   * number of milliseconds as specified in {@link Thread#sleep(long)}.
+   *
+   * @param millis length of time to sleep in milliseconds
+   * @throws InterruptedException if any thread has interrupted the current thread
+   */
+  void sleep(long millis) throws InterruptedException;
+
+  /** Provides the default implementation based on {@link Thread#sleep(long)}. */
+  Sleeper DEFAULT = new Sleeper() {
+
+    public void sleep(long millis) throws InterruptedException {
+      Thread.sleep(millis);
+    }
+  };
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/SdkCoreApiSurfaceTest.java b/sdks/java/core/src/test/java/org/apache/beam/SdkCoreApiSurfaceTest.java
index 26dd9f90f2ca..080f34aa5e68 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/SdkCoreApiSurfaceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/SdkCoreApiSurfaceTest.java
@@ -38,7 +38,6 @@ public void testSdkApiSurface() throws Exception {
     final Set<String> allowed =
         ImmutableSet.of(
             "org.apache.beam",
-            "com.google.api.client",
             "com.fasterxml.jackson.annotation",
             "com.fasterxml.jackson.core",
             "com.fasterxml.jackson.databind",
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java
index 1762d0dd1e40..d307bedc5518 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java
@@ -38,7 +38,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/** Tests for {@link FastNanoClockAndSleeper}. */
+/** Tests for {@link ExpectedLogs}. */
 @RunWith(JUnit4.class)
 public class ExpectedLogsTest {
   private static final Logger LOG = LoggerFactory.getLogger(ExpectedLogsTest.class);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
index 4ee67501817c..80f02e0c8cff 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java
@@ -20,7 +20,6 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
 
-import com.google.api.client.util.BackOff;
 import com.google.common.io.Files;
 import java.io.File;
 import java.io.IOException;
@@ -43,14 +42,10 @@ public class FileChecksumMatcherTest {
   public TemporaryFolder tmpFolder = new TemporaryFolder();
   @Rule
   public ExpectedException thrown = ExpectedException.none();
-  @Rule
-  public FastNanoClockAndSleeper fastClock = new FastNanoClockAndSleeper();
 
   @Mock
   private PipelineResult pResult = Mockito.mock(PipelineResult.class);
 
-  private BackOff backOff = FileChecksumMatcher.BACK_OFF_FACTORY.backoff();
-
   @Test
   public void testPreconditionChecksumIsNull() throws IOException {
     String tmpPath = tmpFolder.newFile().getPath();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SystemNanoTimeSleeper.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SystemNanoTimeSleeper.java
index 3677e847a6a5..dd576698c780 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SystemNanoTimeSleeper.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SystemNanoTimeSleeper.java
@@ -17,9 +17,9 @@
  */
 package org.apache.beam.sdk.testing;
 
-import com.google.api.client.util.Sleeper;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.LockSupport;
+import org.apache.beam.sdk.util.Sleeper;
 
 /**
  * This class provides an expensive sleeper to deal with issues around Java's
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FluentBackoffTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FluentBackoffTest.java
index 20b03cf603a3..e8102788716b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FluentBackoffTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FluentBackoffTest.java
@@ -26,7 +26,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 
-import com.google.api.client.util.BackOff;
 import java.io.IOException;
 import org.joda.time.Duration;
 import org.junit.Rule;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
index 43a91665c9c3..cf8c7224d5fa 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java
@@ -25,16 +25,13 @@
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.spy;
 
-import com.google.api.client.util.BackOff;
 import com.google.common.io.Files;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Pattern;
-import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.io.LocalResources;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -42,17 +39,18 @@
 import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
-import org.mockito.Mock;
-import org.mockito.Mockito;
 
 /** Tests for {@link NumberedShardedFile}. */
 @RunWith(JUnit4.class)
 public class NumberedShardedFileTest {
   @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
   @Rule public ExpectedException thrown = ExpectedException.none();
-  @Rule public FastNanoClockAndSleeper fastClock = new FastNanoClockAndSleeper();
-
-  @Mock private PipelineResult pResult = Mockito.mock(PipelineResult.class);
+  private Sleeper fastClock = new Sleeper() {
+    @Override
+    public void sleep(long millis) throws InterruptedException {
+      // No sleep.
+    }
+  };
 
   private final BackOff backOff = NumberedShardedFile.BACK_OFF_FACTORY.backoff();
   private String filePattern;
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index a4128e803505..985520f03a00 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -53,6 +53,7 @@
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.InstanceBuilder;
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
@@ -319,7 +320,7 @@ private static long getProjectNumber(
       return getProjectNumber(
           projectId,
           crmClient,
-          BACKOFF_FACTORY.backoff(),
+          BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff()),
           Sleeper.DEFAULT);
     }
 
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/BackOffAdapter.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/BackOffAdapter.java
new file mode 100644
index 000000000000..e5a0a6ed4fde
--- /dev/null
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/BackOffAdapter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import java.io.IOException;
+
+/**
+ * An adapter for converting between Apache Beam and Google API client representations of backoffs.
+ */
+public class BackOffAdapter {
+  /**
+   * Returns an adapter to convert from {@link BackOff} to
+   * {@link com.google.api.client.util.BackOff}.
+   */
+  public static com.google.api.client.util.BackOff toGcpBackOff(final BackOff backOff) {
+    return new com.google.api.client.util.BackOff() {
+      @Override
+      public void reset() throws IOException {
+        backOff.reset();
+      }
+
+      @Override
+      public long nextBackOffMillis() throws IOException {
+        return backOff.nextBackOffMillis();
+      }
+    };
+  }
+}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
index ee2e231d3ac8..18e3e2bce080 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
@@ -306,6 +306,9 @@ Integer getUploadBufferSizeBytes() {
     return uploadBufferSizeBytes;
   }
 
+  private static BackOff createBackOff() {
+    return BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff());
+  }
   /**
    * Returns the file size from GCS or throws {@link FileNotFoundException}
    * if the resource does not exist.
@@ -318,7 +321,7 @@ public long fileSize(GcsPath path) throws IOException {
    * Returns the {@link StorageObject} for the given {@link GcsPath}.
    */
   public StorageObject getObject(GcsPath gcsPath) throws IOException {
-    return getObject(gcsPath, BACKOFF_FACTORY.backoff(), Sleeper.DEFAULT);
+    return getObject(gcsPath, createBackOff(), Sleeper.DEFAULT);
   }
 
   @VisibleForTesting
@@ -377,7 +380,7 @@ public Objects listObjects(String bucket, String prefix, @Nullable String pageTo
     try {
       return ResilientOperation.retry(
           ResilientOperation.getGoogleRequestCallable(listObject),
-          BACKOFF_FACTORY.backoff(),
+          createBackOff(),
           RetryDeterminer.SOCKET_ERRORS,
           IOException.class);
     } catch (Exception e) {
@@ -469,7 +472,7 @@ public WritableByteChannel create(GcsPath path, String type, Integer uploadBuffe
   public boolean bucketAccessible(GcsPath path) throws IOException {
     return bucketAccessible(
         path,
-        BACKOFF_FACTORY.backoff(),
+        createBackOff(),
         Sleeper.DEFAULT);
   }
 
@@ -482,7 +485,7 @@ public boolean bucketAccessible(GcsPath path) throws IOException {
   public long bucketOwner(GcsPath path) throws IOException {
     return getBucket(
         path,
-        BACKOFF_FACTORY.backoff(),
+        createBackOff(),
         Sleeper.DEFAULT).getProjectNumber().longValue();
   }
 
@@ -492,7 +495,7 @@ public long bucketOwner(GcsPath path) throws IOException {
    */
   public void createBucket(String projectId, Bucket bucket) throws IOException {
     createBucket(
-        projectId, bucket, BACKOFF_FACTORY.backoff(), Sleeper.DEFAULT);
+        projectId, bucket, createBackOff(), Sleeper.DEFAULT);
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UploadIdResponseInterceptor.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/UploadIdResponseInterceptor.java
similarity index 100%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/UploadIdResponseInterceptor.java
rename to sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/UploadIdResponseInterceptor.java
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FastNanoClockAndSleeper.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/FastNanoClockAndSleeper.java
similarity index 94%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FastNanoClockAndSleeper.java
rename to sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/FastNanoClockAndSleeper.java
index 6bfafa5a8637..f1392d786a9e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FastNanoClockAndSleeper.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/FastNanoClockAndSleeper.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.testing;
+package org.apache.beam.sdk.util;
 
 import com.google.api.client.util.NanoClock;
 import com.google.api.client.util.Sleeper;
@@ -37,7 +37,7 @@ public long nanoTime() {
 
   @Override
   protected void before() throws Throwable {
-    fastNanoTime = NanoClock.SYSTEM.nanoTime();
+    fastNanoTime = SYSTEM.nanoTime();
   }
 
   @Override
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FastNanoClockAndSleeperTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/FastNanoClockAndSleeperTest.java
similarity index 98%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FastNanoClockAndSleeperTest.java
rename to sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/FastNanoClockAndSleeperTest.java
index 7d209515421b..03f958850e9c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FastNanoClockAndSleeperTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/FastNanoClockAndSleeperTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.testing;
+package org.apache.beam.sdk.util;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
index 6ffcaeb816c6..0af584e74e67 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
@@ -76,7 +76,6 @@
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
 import org.apache.beam.sdk.util.GcsUtil.StorageObjectOrIOException;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.junit.Rule;
@@ -374,7 +373,8 @@ public void testRetryFileSizeNonBatch() throws IOException {
     Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
     Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.withMaxRetries(2).backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(
+        FluentBackoff.DEFAULT.withMaxRetries(2).backoff());
 
     when(mockStorage.objects()).thenReturn(mockStorageObjects);
     when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
@@ -491,7 +491,7 @@ public void testCreateBucket() throws IOException {
 
     Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
 
     when(mockStorage.buckets()).thenReturn(mockStorageObjects);
     when(mockStorageObjects.insert(
@@ -514,7 +514,7 @@ public void testCreateBucketAccessErrors() throws IOException {
     Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
     Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());
     GoogleJsonResponseException expectedException =
         googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN,
             "Waves hand mysteriously", "These aren't the buckets you're looking for");
@@ -541,7 +541,8 @@ public void testBucketAccessible() throws IOException {
     Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
     Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(
+        FluentBackoff.DEFAULT.backoff());
 
     when(mockStorage.buckets()).thenReturn(mockStorageObjects);
     when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
@@ -564,7 +565,8 @@ public void testBucketDoesNotExistBecauseOfAccessError() throws IOException {
     Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
     Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(
+        FluentBackoff.DEFAULT.backoff());
     GoogleJsonResponseException expectedException =
         googleJsonResponseException(HttpStatusCodes.STATUS_CODE_FORBIDDEN,
             "Waves hand mysteriously", "These aren't the buckets you're looking for");
@@ -589,7 +591,8 @@ public void testBucketDoesNotExist() throws IOException {
     Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
     Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(
+        FluentBackoff.DEFAULT.backoff());
 
     when(mockStorage.buckets()).thenReturn(mockStorageObjects);
     when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
@@ -612,7 +615,8 @@ public void testGetBucket() throws IOException {
     Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
     Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(
+        FluentBackoff.DEFAULT.backoff());
 
     when(mockStorage.buckets()).thenReturn(mockStorageObjects);
     when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
@@ -635,7 +639,8 @@ public void testGetBucketNotExists() throws IOException {
     Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
     Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);
 
-    BackOff mockBackOff = FluentBackoff.DEFAULT.backoff();
+    BackOff mockBackOff = BackOffAdapter.toGcpBackOff(
+        FluentBackoff.DEFAULT.backoff());
 
     when(mockStorage.buckets()).thenReturn(mockStorageObjects);
     when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/UploadIdResponseInterceptorTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/UploadIdResponseInterceptorTest.java
similarity index 100%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/UploadIdResponseInterceptorTest.java
rename to sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/UploadIdResponseInterceptorTest.java
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index b348abdd22df..5d5a519abf4d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -65,6 +65,7 @@
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
 import org.apache.beam.sdk.util.Transport;
@@ -93,6 +94,9 @@ class BigQueryServicesImpl implements BigQueryServices {
   // The initial backoff for polling the status of a BigQuery job.
   private static final Duration INITIAL_JOB_STATUS_POLL_BACKOFF = Duration.standardSeconds(1);
 
+  private static final FluentBackoff DEFAULT_BACKOFF_FACTORY =
+      FluentBackoff.DEFAULT.withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF);
+
   @Override
   public JobService getJobService(BigQueryOptions options) {
     return new JobServiceImpl(options);
@@ -114,6 +118,10 @@ public BigQueryJsonReader getReaderFromQuery(
     return BigQueryJsonReaderImpl.fromQuery(bqOptions, projectId, queryConfig);
   }
 
+  private static BackOff createDefaultBackoff() {
+    return BackOffAdapter.toGcpBackOff(DEFAULT_BACKOFF_FACTORY.backoff());
+  }
+
   @VisibleForTesting
   static class JobServiceImpl implements BigQueryServices.JobService {
     private final ApiErrorExtractor errorExtractor;
@@ -205,10 +213,7 @@ public void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConf
     private static void startJob(Job job,
       ApiErrorExtractor errorExtractor,
       Bigquery client) throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
-      startJob(job, errorExtractor, client, Sleeper.DEFAULT, backoff);
+      startJob(job, errorExtractor, client, Sleeper.DEFAULT, createDefaultBackoff());
     }
 
     @VisibleForTesting
@@ -249,11 +254,12 @@ static void startJob(
     @Override
     public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException {
       BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(maxAttempts)
-              .withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF)
-              .withMaxBackoff(Duration.standardMinutes(1))
-              .backoff();
+          BackOffAdapter.toGcpBackOff(
+              FluentBackoff.DEFAULT
+                  .withMaxRetries(maxAttempts)
+                  .withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF)
+                  .withMaxBackoff(Duration.standardMinutes(1))
+                  .backoff());
       return pollJob(jobRef, Sleeper.DEFAULT, backoff);
     }
 
@@ -299,16 +305,13 @@ public JobStatistics dryRunQuery(String projectId, JobConfigurationQuery queryCo
           .setConfiguration(new JobConfiguration()
               .setQuery(queryConfig)
               .setDryRun(true));
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
       return executeWithRetries(
           client.jobs().insert(projectId, job),
           String.format(
               "Unable to dry run query: %s, aborting after %d retries.",
               queryConfig, MAX_RPC_RETRIES),
           Sleeper.DEFAULT,
-          backoff,
+          createDefaultBackoff(),
           ALWAYS_RETRY).getStatistics();
     }
 
@@ -321,10 +324,7 @@ public JobStatistics dryRunQuery(String projectId, JobConfigurationQuery queryCo
      */
     @Override
     public Job getJob(JobReference jobRef) throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
-     return getJob(jobRef, Sleeper.DEFAULT, backoff);
+     return getJob(jobRef, Sleeper.DEFAULT, createDefaultBackoff());
     }
 
     @VisibleForTesting
@@ -371,7 +371,7 @@ static class DatasetServiceImpl implements DatasetService {
         FluentBackoff.DEFAULT.withInitialBackoff(Duration.millis(200)).withMaxRetries(5);
 
     // A backoff for rate limit exceeded errors. Retries forever.
-    private static final FluentBackoff DEFAULT_BACKOFF_FACTORY =
+    private static final FluentBackoff RATE_LIMIT_BACKOFF_FACTORY =
         FluentBackoff.DEFAULT
             .withInitialBackoff(Duration.standardSeconds(1))
             .withMaxBackoff(Duration.standardMinutes(2));
@@ -420,10 +420,7 @@ private DatasetServiceImpl(BigQueryOptions bqOptions) {
     @Nullable
     public Table getTable(TableReference tableRef)
         throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
-      return getTable(tableRef, backoff, Sleeper.DEFAULT);
+      return getTable(tableRef, createDefaultBackoff(), Sleeper.DEFAULT);
     }
 
     @VisibleForTesting
@@ -528,9 +525,6 @@ Table tryCreateTable(Table table, BackOff backoff, Sleeper sleeper)
      */
     @Override
     public void deleteTable(TableReference tableRef) throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
       executeWithRetries(
           client.tables().delete(
               tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()),
@@ -538,16 +532,13 @@ public void deleteTable(TableReference tableRef) throws IOException, Interrupted
               "Unable to delete table: %s, aborting after %d retries.",
               tableRef.getTableId(), MAX_RPC_RETRIES),
           Sleeper.DEFAULT,
-          backoff,
+          createDefaultBackoff(),
           ALWAYS_RETRY);
     }
 
     @Override
     public boolean isTableEmpty(TableReference tableRef) throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
-      return isTableEmpty(tableRef, backoff, Sleeper.DEFAULT);
+      return isTableEmpty(tableRef, createDefaultBackoff(), Sleeper.DEFAULT);
     }
 
     @VisibleForTesting
@@ -575,16 +566,13 @@ boolean isTableEmpty(TableReference tableRef, BackOff backoff, Sleeper sleeper)
     @Override
     public Dataset getDataset(String projectId, String datasetId)
         throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
       return executeWithRetries(
           client.datasets().get(projectId, datasetId),
           String.format(
               "Unable to get dataset: %s, aborting after %d retries.",
               datasetId, MAX_RPC_RETRIES),
           Sleeper.DEFAULT,
-          backoff,
+          createDefaultBackoff(),
           DONT_RETRY_NOT_FOUND);
     }
 
@@ -599,10 +587,8 @@ public Dataset getDataset(String projectId, String datasetId)
     public void createDataset(
         String projectId, String datasetId, @Nullable String location, @Nullable String description)
         throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
-      createDataset(projectId, datasetId, location, description, Sleeper.DEFAULT, backoff);
+      createDataset(
+          projectId, datasetId, location, description, Sleeper.DEFAULT, createDefaultBackoff());
     }
 
     private void createDataset(
@@ -659,16 +645,13 @@ private void createDataset(
     @Override
     public void deleteDataset(String projectId, String datasetId)
         throws IOException, InterruptedException {
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
       executeWithRetries(
           client.datasets().delete(projectId, datasetId),
           String.format(
               "Unable to delete table: %s, aborting after %d retries.",
               datasetId, MAX_RPC_RETRIES),
           Sleeper.DEFAULT,
-          backoff,
+          createDefaultBackoff(),
           ALWAYS_RETRY);
     }
 
@@ -725,7 +708,9 @@ long insertAll(TableReference ref, List<TableRow> rowList, @Nullable List<String
                 executor.submit(new Callable<List<TableDataInsertAllResponse.InsertErrors>>() {
                   @Override
                   public List<TableDataInsertAllResponse.InsertErrors> call() throws IOException {
-                    BackOff backoff = DEFAULT_BACKOFF_FACTORY.backoff();
+                    // A backoff for rate limit exceeded errors. Retries forever.
+                    BackOff backoff = BackOffAdapter.toGcpBackOff(
+                        RATE_LIMIT_BACKOFF_FACTORY.backoff());
                     while (true) {
                       try {
                         return insert.execute().getInsertErrors();
@@ -811,7 +796,10 @@ public long insertAll(
         TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList)
         throws IOException, InterruptedException {
       return insertAll(
-          ref, rowList, insertIdList, INSERT_BACKOFF_FACTORY.backoff(), Sleeper.DEFAULT);
+          ref, rowList, insertIdList,
+          BackOffAdapter.toGcpBackOff(
+              INSERT_BACKOFF_FACTORY.backoff()),
+          Sleeper.DEFAULT);
     }
 
 
@@ -822,9 +810,6 @@ public Table patchTableDescription(TableReference tableReference,
       Table table = new Table();
       table.setDescription(tableDescription);
 
-      BackOff backoff =
-          FluentBackoff.DEFAULT
-              .withMaxRetries(MAX_RPC_RETRIES).withInitialBackoff(INITIAL_RPC_BACKOFF).backoff();
       return executeWithRetries(
           client.tables().patch(
               tableReference.getProjectId(),
@@ -835,7 +820,7 @@ public Table patchTableDescription(TableReference tableReference,
               "Unable to patch table description: %s, aborting after %d retries.",
               tableReference, MAX_RPC_RETRIES),
           Sleeper.DEFAULT,
-          backoff,
+          createDefaultBackoff(),
           ALWAYS_RETRY);
     }
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java
index 59f2bb682e60..ba19cf012aae 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableRowIterator.java
@@ -57,6 +57,8 @@
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
@@ -453,8 +455,9 @@ private static <T> T executeWithBackOff(AbstractGoogleClientRequest<T> client, S
       throws IOException, InterruptedException {
     Sleeper sleeper = Sleeper.DEFAULT;
     BackOff backOff =
-        FluentBackoff.DEFAULT
-            .withMaxRetries(MAX_RETRIES).withInitialBackoff(INITIAL_BACKOFF_TIME).backoff();
+        BackOffAdapter.toGcpBackOff(
+            FluentBackoff.DEFAULT
+                .withMaxRetries(MAX_RETRIES).withInitialBackoff(INITIAL_BACKOFF_TIME).backoff());
 
     T result = null;
     while (true) {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index d6464dd95855..16bb1b4d36a1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -32,9 +32,6 @@
 import static com.google.datastore.v1.client.DatastoreHelper.makeValue;
 
 import com.google.api.client.http.HttpRequestInitializer;
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.BackOffUtils;
-import com.google.api.client.util.Sleeper;
 import com.google.auth.Credentials;
 import com.google.auth.http.HttpCredentialsAdapter;
 import com.google.auto.value.AutoValue;
@@ -89,8 +86,11 @@
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.apache.beam.sdk.util.BackOff;
+import org.apache.beam.sdk.util.BackOffUtils;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
+import org.apache.beam.sdk.util.Sleeper;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
index ef51650633c1..b41490f34a38 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
@@ -67,7 +67,8 @@
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl.JobServiceImpl;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.ExpectedLogs;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
+import org.apache.beam.sdk.util.BackOffAdapter;
+import org.apache.beam.sdk.util.FastNanoClockAndSleeper;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
 import org.apache.beam.sdk.util.Transport;
@@ -133,7 +134,8 @@ public void testStartLoadJobSucceeds() throws IOException, InterruptedException
 
     Sleeper sleeper = new FastNanoClockAndSleeper();
     JobServiceImpl.startJob(
-        testJob, new ApiErrorExtractor(), bigquery, sleeper, FluentBackoff.DEFAULT.backoff());
+        testJob, new ApiErrorExtractor(), bigquery, sleeper,
+        BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()));
 
     verify(response, times(1)).getStatusCode();
     verify(response, times(1)).getContent();
@@ -157,7 +159,8 @@ public void testStartLoadJobSucceedsAlreadyExists() throws IOException, Interrup
 
     Sleeper sleeper = new FastNanoClockAndSleeper();
     JobServiceImpl.startJob(
-        testJob, new ApiErrorExtractor(), bigquery, sleeper, FluentBackoff.DEFAULT.backoff());
+        testJob, new ApiErrorExtractor(), bigquery, sleeper,
+        BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()));
 
     verify(response, times(1)).getStatusCode();
     verify(response, times(1)).getContent();
@@ -185,7 +188,8 @@ public void testStartLoadJobRetry() throws IOException, InterruptedException {
 
     Sleeper sleeper = new FastNanoClockAndSleeper();
     JobServiceImpl.startJob(
-        testJob, new ApiErrorExtractor(), bigquery, sleeper, FluentBackoff.DEFAULT.backoff());
+        testJob, new ApiErrorExtractor(), bigquery, sleeper,
+        BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()));
 
     verify(response, times(2)).getStatusCode();
     verify(response, times(2)).getContent();
@@ -500,7 +504,8 @@ public void testInsertRetry() throws Exception {
 
     DatasetServiceImpl dataService =
         new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
-    dataService.insertAll(ref, rows, null, TEST_BACKOFF.backoff(), new MockSleeper());
+    dataService.insertAll(ref, rows, null,
+        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
     verify(response, times(2)).getStatusCode();
     verify(response, times(2)).getContent();
     verify(response, times(2)).getContentType();
@@ -536,7 +541,8 @@ public void testInsertRetrySelectRows() throws Exception {
 
     DatasetServiceImpl dataService =
         new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
-    dataService.insertAll(ref, rows, insertIds, TEST_BACKOFF.backoff(), new MockSleeper());
+    dataService.insertAll(ref, rows, insertIds,
+        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
     verify(response, times(2)).getStatusCode();
     verify(response, times(2)).getContent();
     verify(response, times(2)).getContentType();
@@ -577,7 +583,8 @@ public InputStream answer(InvocationOnMock invocation) throws Throwable {
 
     // Expect it to fail.
     try {
-      dataService.insertAll(ref, rows, null, TEST_BACKOFF.backoff(), new MockSleeper());
+      dataService.insertAll(ref, rows, null,
+          BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
       fail();
     } catch (IOException e) {
       assertThat(e, instanceOf(IOException.class));
@@ -617,7 +624,8 @@ public void testInsertDoesNotRetry() throws Throwable {
         new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     try {
-      dataService.insertAll(ref, rows, null, TEST_BACKOFF.backoff(), new MockSleeper());
+      dataService.insertAll(ref, rows, null,
+          BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper());
       fail();
     } catch (RuntimeException e) {
       verify(response, times(1)).getStatusCode();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
index ed6a0beff703..2045bb7bda26 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FakeJobService.java
@@ -66,6 +66,7 @@
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.Transport;
@@ -187,11 +188,12 @@ public void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConf
   public Job pollJob(JobReference jobRef, int maxAttempts)
       throws InterruptedException {
     BackOff backoff =
-        FluentBackoff.DEFAULT
-            .withMaxRetries(maxAttempts)
-            .withInitialBackoff(Duration.millis(10))
-            .withMaxBackoff(Duration.standardSeconds(1))
-            .backoff();
+        BackOffAdapter.toGcpBackOff(
+            FluentBackoff.DEFAULT
+                .withMaxRetries(maxAttempts)
+                .withInitialBackoff(Duration.millis(10))
+                .withMaxBackoff(Duration.standardSeconds(1))
+                .backoff());
     Sleeper sleeper = Sleeper.DEFAULT;
     try {
       do {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
index 85c27dd3d721..dc91638878dd 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
@@ -26,9 +26,6 @@
 import static com.google.datastore.v1.client.DatastoreHelper.makeValue;
 
 import com.google.api.client.http.HttpRequestInitializer;
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.BackOffUtils;
-import com.google.api.client.util.Sleeper;
 import com.google.auth.Credentials;
 import com.google.auth.http.HttpCredentialsAdapter;
 import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
@@ -57,8 +54,11 @@
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.util.BackOff;
+import org.apache.beam.sdk.util.BackOffUtils;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
+import org.apache.beam.sdk.util.Sleeper;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java
index 2bdfffafcd80..6dc810ba329c 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java
@@ -49,6 +49,7 @@
 import javax.annotation.concurrent.NotThreadSafe;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.testing.SerializableMatcher;
+import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.Transport;
 import org.hamcrest.Description;
@@ -118,7 +119,8 @@ protected boolean matchesSafely(PipelineResult pipelineResult) {
       queryContent.setQuery(query);
 
       response = queryWithRetries(
-          bigqueryClient, queryContent, Sleeper.DEFAULT, BACKOFF_FACTORY.backoff());
+          bigqueryClient, queryContent, Sleeper.DEFAULT,
+          BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff()));
     } catch (IOException | InterruptedException e) {
       if (e instanceof InterruptedIOException) {
         Thread.currentThread().interrupt();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java
index 5fcdce9cf217..2b03909c0a52 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java
@@ -36,7 +36,8 @@
 import java.io.IOException;
 import java.math.BigInteger;
 import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
+import org.apache.beam.sdk.util.BackOffAdapter;
+import org.apache.beam.sdk.util.FastNanoClockAndSleeper;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -131,7 +132,7 @@ public void testQueryWithRetriesWhenServiceFails() throws Exception {
           mockBigqueryClient,
           new QueryRequest(),
           fastClock,
-          BigqueryMatcher.BACKOFF_FACTORY.backoff());
+          BackOffAdapter.toGcpBackOff(BigqueryMatcher.BACKOFF_FACTORY.backoff()));
     } finally {
       verify(mockJobs, atLeast(BigqueryMatcher.MAX_QUERY_RETRIES))
           .query(eq(projectId), eq(new QueryRequest()));
@@ -151,7 +152,7 @@ public void testQueryWithRetriesWhenQueryResponseNull() throws Exception {
           mockBigqueryClient,
           new QueryRequest(),
           fastClock,
-          BigqueryMatcher.BACKOFF_FACTORY.backoff());
+          BackOffAdapter.toGcpBackOff(BigqueryMatcher.BACKOFF_FACTORY.backoff()));
     } finally {
       verify(mockJobs, atLeast(BigqueryMatcher.MAX_QUERY_RETRIES))
           .query(eq(projectId), eq(new QueryRequest()));

From 16f355f7e481ebf029c0edb878742f6fea57b6cd Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Mon, 8 May 2017 15:37:29 -0700
Subject: [PATCH 124/387] [BEAM-2211] DataflowRunner: remove validation of file
 read/write paths

Now that users can implement and register custom FileSystems,
we can no longer really effectively validate filesystems they
can read or write files from. They can even register file://
to point to some HDFS path, e.g.,
---
 .../beam/runners/dataflow/DataflowRunner.java | 60 -------------------
 .../beam/runners/dataflow/ReadTranslator.java | 12 ----
 .../DataflowPipelineTranslatorTest.java       | 26 --------
 .../runners/dataflow/DataflowRunnerTest.java  | 56 -----------------
 4 files changed, 154 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 5278a4a576d0..250c0641e426 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -84,13 +84,10 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.FileBasedSink;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.UnboundedSource;
-import org.apache.beam.sdk.io.WriteFiles;
 import org.apache.beam.sdk.io.fs.PathValidator;
-import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder;
@@ -98,7 +95,6 @@
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubUnboundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
@@ -343,10 +339,6 @@ private List<PTransformOverride> getOverrides(boolean streaming) {
                   PTransformMatchers.stateOrTimerParDoSingle(),
                   BatchStatefulParDoOverrides.singleOutputOverrideFactory()))
 
-          // WriteFiles uses views internally
-          .add(
-              PTransformOverride.of(
-                  PTransformMatchers.classEqualTo(WriteFiles.class), new BatchWriteFactory(this)))
           .add(
               PTransformOverride.of(
                   PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
@@ -805,58 +797,6 @@ void recordViewUsesNonDeterministicKeyCoder(PTransform<?, ?> ptransform) {
     ptransformViewsWithNonDeterministicKeyCoders.add(ptransform);
   }
 
-  private class BatchWriteFactory<T>
-      implements PTransformOverrideFactory<PCollection<T>, PDone, WriteFiles<T>> {
-    private final DataflowRunner runner;
-    private BatchWriteFactory(DataflowRunner dataflowRunner) {
-      this.runner = dataflowRunner;
-    }
-
-    @Override
-    public PTransformReplacement<PCollection<T>, PDone> getReplacementTransform(
-        AppliedPTransform<PCollection<T>, PDone, WriteFiles<T>> transform) {
-      return PTransformReplacement.of(
-          PTransformReplacements.getSingletonMainInput(transform),
-          new BatchWrite<>(runner, transform.getTransform()));
-    }
-
-    @Override
-    public Map<PValue, ReplacementOutput> mapOutputs(
-        Map<TupleTag<?>, PValue> outputs, PDone newOutput) {
-      return Collections.emptyMap();
-    }
-  }
-
-  /**
-   * Specialized implementation which overrides
-   * {@link WriteFiles WriteFiles} to provide Google
-   * Cloud Dataflow specific path validation of {@link FileBasedSink}s.
-   */
-  private static class BatchWrite<T> extends PTransform<PCollection<T>, PDone> {
-    private final DataflowRunner runner;
-    private final WriteFiles<T> transform;
-    /**
-     * Builds an instance of this class from the overridden transform.
-     */
-    @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
-    public BatchWrite(DataflowRunner runner, WriteFiles<T> transform) {
-      this.runner = runner;
-      this.transform = transform;
-    }
-
-    @Override
-    public PDone expand(PCollection<T> input) {
-      ValueProvider<ResourceId> outputDirectory =
-          transform.getSink().getBaseOutputDirectoryProvider();
-      if (outputDirectory.isAccessible()) {
-        PathValidator validator = runner.options.getPathValidator();
-        validator.validateOutputResourceSupported(
-            outputDirectory.get().resolve("some-file", StandardResolveOptions.RESOLVE_FILE));
-      }
-      return transform.expand(input);
-    }
-  }
-
   // ================================================================================
   // PubsubIO translations
   // ================================================================================
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
index 396c3051e5f8..30ecbf581d2a 100755
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
@@ -26,10 +26,8 @@
 import java.util.Map;
 import org.apache.beam.runners.dataflow.internal.CustomSources;
 import org.apache.beam.runners.dataflow.util.PropertyNames;
-import org.apache.beam.sdk.io.FileBasedSource;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.Source;
-import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PValue;
 
@@ -46,16 +44,6 @@ public static <T> void translateReadHelper(Source<T> source,
       PTransform<?, ? extends PValue> transform,
       TranslationContext context) {
     try {
-      if (source instanceof FileBasedSource) {
-        ValueProvider<String> filePatternOrSpec =
-            ((FileBasedSource<?>) source).getFileOrPatternSpecProvider();
-        if (filePatternOrSpec.isAccessible()) {
-          context.getPipelineOptions()
-              .getPathValidator()
-              .validateInputFilePatternSupported(filePatternOrSpec.get());
-        }
-      }
-
       StepTranslationContext stepContext = context.addStep(transform, "ParallelRead");
       stepContext.addInput(PropertyNames.FORMAT, PropertyNames.CUSTOM_SOURCE_FORMAT);
       stepContext.addInput(
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 48cb79f1731d..9040f8f01c4f 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -20,13 +20,10 @@
 import static org.apache.beam.runners.dataflow.util.Structs.addObject;
 import static org.apache.beam.runners.dataflow.util.Structs.getDictionary;
 import static org.apache.beam.runners.dataflow.util.Structs.getString;
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.core.IsInstanceOf.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
@@ -109,7 +106,6 @@
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.internal.matchers.ThrowableMessageMatcher;
 import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -809,28 +805,6 @@ private void applyRead(Pipeline pipeline, String path) {
     pipeline.apply("Read(" + path + ")", TextIO.read().from(path));
   }
 
-  /**
-   * Recursive wildcards are not supported.
-   * This tests "**".
-   */
-  @Test
-  public void testBadWildcardRecursive() throws Exception {
-    DataflowPipelineOptions options = buildPipelineOptions();
-    Pipeline pipeline = Pipeline.create(options);
-    DataflowPipelineTranslator t = DataflowPipelineTranslator.fromOptions(options);
-
-    pipeline.apply(TextIO.read().from("gs://bucket/foo**/baz"));
-
-    // Check that translation does fail.
-    thrown.expectCause(allOf(
-        instanceOf(IllegalArgumentException.class),
-        ThrowableMessageMatcher.hasMessage(containsString("Unsupported wildcard usage"))));
-    t.translate(
-        pipeline,
-        DataflowRunner.fromOptions(options),
-        Collections.<DataflowPackage>emptyList());
-  }
-
   private static class TestValueProvider implements ValueProvider<String>, Serializable {
     @Override
     public boolean isAccessible() {
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index 3d477261fd7a..6848b85cfa33 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -22,7 +22,6 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.startsWith;
 import static org.junit.Assert.assertEquals;
@@ -97,7 +96,6 @@
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.internal.matchers.ThrowableMessageMatcher;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
@@ -587,60 +585,6 @@ public void testGcsStagingLocationInitialization() throws Exception {
     assertNotNull(options.getStagingLocation());
   }
 
-  @Test
-  public void testNonGcsFilePathInReadFailure() throws IOException {
-    Pipeline p = buildDataflowPipeline(buildPipelineOptions());
-    p.apply("ReadMyNonGcsFile", TextIO.read().from(tmpFolder.newFile().getPath()));
-
-    thrown.expectCause(Matchers.allOf(
-        instanceOf(IllegalArgumentException.class),
-        ThrowableMessageMatcher.hasMessage(
-            containsString("Expected a valid 'gs://' path but was given"))));
-    p.run();
-
-    ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
-    Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
-    assertValidJob(jobCaptor.getValue());
-  }
-
-  @Test
-  public void testNonGcsFilePathInWriteFailure() throws IOException {
-    Pipeline p = buildDataflowPipeline(buildPipelineOptions());
-
-    p.apply("ReadMyGcsFile", TextIO.read().from("gs://bucket/object"))
-        .apply("WriteMyNonGcsFile", TextIO.write().to("/tmp/file"));
-
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage(containsString("Expected a valid 'gs://' path but was given"));
-    p.run();
-  }
-
-  @Test
-  public void testMultiSlashGcsFileReadPath() throws IOException {
-    Pipeline p = buildDataflowPipeline(buildPipelineOptions());
-    p.apply("ReadInvalidGcsFile", TextIO.read().from("gs://bucket/tmp//file"));
-
-    thrown.expectCause(Matchers.allOf(
-        instanceOf(IllegalArgumentException.class),
-        ThrowableMessageMatcher.hasMessage(containsString("consecutive slashes"))));
-    p.run();
-
-    ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
-    Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
-    assertValidJob(jobCaptor.getValue());
-  }
-
-  @Test
-  public void testMultiSlashGcsFileWritePath() throws IOException {
-    Pipeline p = buildDataflowPipeline(buildPipelineOptions());
-    PCollection<String> pc = p.apply("ReadMyGcsFile", TextIO.read().from("gs://bucket/object"));
-    pc.apply("WriteInvalidGcsFile", TextIO.write().to("gs://bucket/tmp//file"));
-
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("consecutive slashes");
-    p.run();
-  }
-
   @Test
   public void testInvalidGcpTempLocation() throws IOException {
     DataflowPipelineOptions options = buildPipelineOptions();

From 27e9a060ed593b2b53b88481591f14b1a274c61b Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Fri, 5 May 2017 16:20:37 -0700
Subject: [PATCH 125/387] Remove explicit used of nested contexts.

    find . -type f -name '*.java' | xargs sed -i '' 's/\([.]..code[(].*\),  *context.nested..[)]/\1)/'
    find . -type f -name '*.java' | xargs sed -i '' 's/\([.]..code[(].*\),  *nestedContext[)]/\1)/'
    find . -type f -name '*.java' | xargs sed -i '' 's/\([.]..code[(].*\),  *Context.NESTED[)]/\1)/'
    find . -type f -name '*.java' | xargs sed -i '' 's/\([.]..code[(].*\),  *[^ ]*.Context.NESTED[)]/\1)/'

Added back explicit context in CoGbkResult.java due to compile error.
---
 .../UnboundedReadFromBoundedSource.java       |  4 +--
 .../core/ElementAndRestrictionCoder.java      |  4 +--
 .../beam/runners/core/KeyedWorkItemCoder.java |  8 +++---
 .../beam/runners/core/TimerInternals.java     | 12 ++++----
 .../types/CoderTypeSerializer.java            |  4 +--
 .../SingletonKeyedWorkItemCoder.java          |  4 +--
 .../state/FlinkKeyGroupStateInternals.java    |  8 +++---
 .../runners/dataflow/BatchViewOverrides.java  |  4 +--
 .../runners/dataflow/internal/IsmFormat.java  | 24 ++++++++--------
 .../spark/aggregators/NamedAggregators.java   |  4 +--
 .../beam/sdk/coders/BigDecimalCoder.java      |  4 +--
 .../beam/sdk/coders/IterableLikeCoder.java    |  8 +++---
 .../org/apache/beam/sdk/coders/KvCoder.java   |  4 +--
 .../org/apache/beam/sdk/coders/MapCoder.java  | 12 ++++----
 .../org/apache/beam/sdk/io/FileBasedSink.java |  4 +--
 .../sdk/transforms/ApproximateQuantiles.java  | 20 ++++++-------
 .../beam/sdk/transforms/CombineFns.java       |  4 +--
 .../org/apache/beam/sdk/transforms/Mean.java  |  4 +--
 .../beam/sdk/transforms/join/CoGbkResult.java |  2 +-
 .../transforms/windowing/IntervalWindow.java  |  4 +--
 .../apache/beam/sdk/util/WindowedValue.java   | 10 +++----
 .../beam/sdk/values/TimestampedValue.java     |  4 +--
 .../beam/sdk/values/ValueInSingleWindow.java  | 12 ++++----
 .../beam/sdk/values/ValueWithRecordId.java    |  4 +--
 .../sdk/coders/SerializableCoderTest.java     | 28 +++++++++----------
 .../beam/sdk/transforms/CombineTest.java      |  4 +--
 .../beam/sdk/transforms/CreateTest.java       |  4 +--
 .../windowing/GlobalWindowTest.java           |  2 +-
 ...fferedElementCountingOutputStreamTest.java |  5 ++--
 .../BeamFnDataBufferingOutboundObserver.java  |  2 +-
 .../data/BeamFnDataInboundObserver.java       |  2 +-
 ...amFnDataBufferingOutboundObserverTest.java |  2 +-
 .../data/BeamFnDataInboundObserverTest.java   |  2 +-
 .../sdk/io/gcp/bigquery/ShardedKeyCoder.java  | 11 ++++----
 .../gcp/bigquery/TableDestinationCoder.java   | 10 +++----
 .../io/gcp/bigquery/TableRowInfoCoder.java    |  4 +--
 .../io/gcp/bigquery/WriteBundlesToFiles.java  | 12 ++++----
 .../PubsubMessageWithAttributesCoder.java     |  4 +--
 .../io/gcp/pubsub/PubsubUnboundedSink.java    | 16 +++++------
 .../io/gcp/pubsub/PubsubUnboundedSource.java  |  2 +-
 .../apache/beam/sdk/io/xml/JAXBCoderTest.java |  8 +++---
 41 files changed, 145 insertions(+), 145 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
index 1424b8be2518..ae28e3adbd3e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
@@ -223,7 +223,7 @@ static class CheckpointCoder<T> extends StructuredCoder<Checkpoint<T>> {
       @Override
       public void encode(Checkpoint<T> value, OutputStream outStream, Context context)
           throws CoderException, IOException {
-        elemsCoder.encode(value.residualElements, outStream, context.nested());
+        elemsCoder.encode(value.residualElements, outStream);
         sourceCoder.encode(value.residualSource, outStream, context);
       }
 
@@ -232,7 +232,7 @@ public void encode(Checkpoint<T> value, OutputStream outStream, Context context)
       public Checkpoint<T> decode(InputStream inStream, Context context)
           throws CoderException, IOException {
         return new Checkpoint<>(
-            elemsCoder.decode(inStream, context.nested()),
+            elemsCoder.decode(inStream),
             sourceCoder.decode(inStream, context));
       }
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
index 83c4e6281d71..5ddd865a518f 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
@@ -55,14 +55,14 @@ public void encode(
     if (value == null) {
       throw new CoderException("cannot encode a null ElementAndRestriction");
     }
-    elementCoder.encode(value.element(), outStream, context.nested());
+    elementCoder.encode(value.element(), outStream);
     restrictionCoder.encode(value.restriction(), outStream, context);
   }
 
   @Override
   public ElementAndRestriction<ElementT, RestrictionT> decode(InputStream inStream, Context context)
       throws IOException {
-    ElementT key = elementCoder.decode(inStream, context.nested());
+    ElementT key = elementCoder.decode(inStream);
     RestrictionT value = restrictionCoder.decode(inStream, context);
     return ElementAndRestriction.of(key, value);
   }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
index e1872b542260..ac8a34cfb9b0 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
@@ -72,8 +72,8 @@ public Coder<ElemT> getElementCoder() {
   public void encode(KeyedWorkItem<K, ElemT> value, OutputStream outStream, Coder.Context context)
       throws CoderException, IOException {
     Coder.Context nestedContext = context.nested();
-    keyCoder.encode(value.key(), outStream, nestedContext);
-    timersCoder.encode(value.timersIterable(), outStream, nestedContext);
+    keyCoder.encode(value.key(), outStream);
+    timersCoder.encode(value.timersIterable(), outStream);
     elemsCoder.encode(value.elementsIterable(), outStream, context);
   }
 
@@ -81,8 +81,8 @@ public void encode(KeyedWorkItem<K, ElemT> value, OutputStream outStream, Coder.
   public KeyedWorkItem<K, ElemT> decode(InputStream inStream, Coder.Context context)
       throws CoderException, IOException {
     Coder.Context nestedContext = context.nested();
-    K key = keyCoder.decode(inStream, nestedContext);
-    Iterable<TimerData> timers = timersCoder.decode(inStream, nestedContext);
+    K key = keyCoder.decode(inStream);
+    Iterable<TimerData> timers = timersCoder.decode(inStream);
     Iterable<WindowedValue<ElemT>> elems = elemsCoder.decode(inStream, context);
     return KeyedWorkItems.workItem(key, timers, elems);
   }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
index 888c11f97313..3607fdd6cb6d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
@@ -241,9 +241,9 @@ private TimerDataCoder(Coder<? extends BoundedWindow> windowCoder) {
     public void encode(TimerData timer, OutputStream outStream, Context context)
         throws CoderException, IOException {
       Context nestedContext = context.nested();
-      STRING_CODER.encode(timer.getTimerId(), outStream, nestedContext);
-      STRING_CODER.encode(timer.getNamespace().stringKey(), outStream, nestedContext);
-      INSTANT_CODER.encode(timer.getTimestamp(), outStream, nestedContext);
+      STRING_CODER.encode(timer.getTimerId(), outStream);
+      STRING_CODER.encode(timer.getNamespace().stringKey(), outStream);
+      INSTANT_CODER.encode(timer.getTimestamp(), outStream);
       STRING_CODER.encode(timer.getDomain().name(), outStream, context);
     }
 
@@ -251,10 +251,10 @@ public void encode(TimerData timer, OutputStream outStream, Context context)
     public TimerData decode(InputStream inStream, Context context)
         throws CoderException, IOException {
       Context nestedContext = context.nested();
-      String timerId = STRING_CODER.decode(inStream, nestedContext);
+      String timerId = STRING_CODER.decode(inStream);
       StateNamespace namespace =
-          StateNamespaces.fromString(STRING_CODER.decode(inStream, nestedContext), windowCoder);
-      Instant timestamp = INSTANT_CODER.decode(inStream, nestedContext);
+          StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder);
+      Instant timestamp = INSTANT_CODER.decode(inStream);
       TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream, context));
       return TimerData.of(timerId, namespace, timestamp, domain);
     }
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java
index e210ed9d7b98..e003119aa96f 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java
@@ -77,14 +77,14 @@ public int getLength() {
   @Override
   public void serialize(T t, DataOutputView dataOutputView) throws IOException {
     DataOutputViewWrapper outputWrapper = new DataOutputViewWrapper(dataOutputView);
-    coder.encode(t, outputWrapper, Coder.Context.NESTED);
+    coder.encode(t, outputWrapper);
   }
 
   @Override
   public T deserialize(DataInputView dataInputView) throws IOException {
     try {
       DataInputViewWrapper inputWrapper = new DataInputViewWrapper(dataInputView);
-      return coder.decode(inputWrapper, Coder.Context.NESTED);
+      return coder.decode(inputWrapper);
     } catch (CoderException e) {
       Throwable cause = e.getCause();
       if (cause instanceof EOFException) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
index f21869341a01..d7bae7e7b3db 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
@@ -70,14 +70,14 @@ public void encode(SingletonKeyedWorkItem<K, ElemT> value,
                      OutputStream outStream,
                      Context context)
       throws CoderException, IOException {
-    keyCoder.encode(value.key(), outStream, context.nested());
+    keyCoder.encode(value.key(), outStream);
     valueCoder.encode(value.value, outStream, context);
   }
 
   @Override
   public SingletonKeyedWorkItem<K, ElemT> decode(InputStream inStream, Context context)
       throws CoderException, IOException {
-    K key = keyCoder.decode(inStream, context.nested());
+    K key = keyCoder.decode(inStream);
     WindowedValue<ElemT> value = valueCoder.decode(inStream, context);
     return new SingletonKeyedWorkItem<>(key, value);
   }
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
index d6af4f9be944..8d437d56d573 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
@@ -430,8 +430,8 @@ public void snapshotKeyGroupState(int keyGroupIdx, DataOutputStream out) throws
       Map<String, ?> map = entry.getValue().f1;
       out.writeInt(map.size());
       for (Map.Entry<String, ?> entry1 : map.entrySet()) {
-        StringUtf8Coder.of().encode(entry1.getKey(), out, Context.NESTED);
-        coder.encode(entry1.getValue(), out, Context.NESTED);
+        StringUtf8Coder.of().encode(entry1.getKey(), out);
+        coder.encode(entry1.getValue(), out);
       }
     }
   }
@@ -463,8 +463,8 @@ public void restoreKeyGroupState(int keyGroupIdx, DataInputStream in,
       Map<String, Object> map = (Map<String, Object>) tuple2.f1;
       int mapSize = in.readInt();
       for (int j = 0; j < mapSize; j++) {
-        String namespace = StringUtf8Coder.of().decode(in, Context.NESTED);
-        Object value = coder.decode(in, Context.NESTED);
+        String namespace = StringUtf8Coder.of().decode(in);
+        Object value = coder.decode(in);
         map.put(namespace, value);
       }
     }
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index ecd0365ed88c..0e60fa09dba9 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -1353,7 +1353,7 @@ public static <K, V1, V2> TransformedMapCoder<K, V1, V2> of(
     @Override
     public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream,
         Coder.Context context) throws CoderException, IOException {
-      transformCoder.encode(value.transform, outStream, context.nested());
+      transformCoder.encode(value.transform, outStream);
       originalMapCoder.encode(value.originalMap, outStream, context);
     }
 
@@ -1361,7 +1361,7 @@ public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream,
     public TransformedMap<K, V1, V2> decode(
         InputStream inStream, Coder.Context context) throws CoderException, IOException {
       return new TransformedMap<>(
-          transformCoder.decode(inStream, context.nested()),
+          transformCoder.decode(inStream),
           originalMapCoder.decode(inStream, context));
     }
 
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
index 00e0c54c2d5f..0f0cd4d940d7 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
@@ -239,12 +239,12 @@ public void encode(IsmRecord<V> value, OutputStream outStream,
             keyComponentCoders.size(), value.getKeyComponents()));
       }
       for (int i = 0; i < keyComponentCoders.size(); ++i) {
-        getKeyComponentCoder(i).encode(value.getKeyComponent(i), outStream, context.nested());
+        getKeyComponentCoder(i).encode(value.getKeyComponent(i), outStream);
       }
       if (isMetadataKey(value.getKeyComponents())) {
-        ByteArrayCoder.of().encode(value.getMetadata(), outStream, context.nested());
+        ByteArrayCoder.of().encode(value.getMetadata(), outStream);
       } else {
-        valueCoder.encode(value.getValue(), outStream, context.nested());
+        valueCoder.encode(value.getValue(), outStream);
       }
     }
 
@@ -253,13 +253,13 @@ public IsmRecord<V> decode(InputStream inStream, Coder.Context context)
         throws CoderException, IOException {
       List<Object> keyComponents = new ArrayList<>(keyComponentCoders.size());
       for (Coder<?> keyCoder : keyComponentCoders) {
-        keyComponents.add(keyCoder.decode(inStream, context.nested()));
+        keyComponents.add(keyCoder.decode(inStream));
       }
       if (isMetadataKey(keyComponents)) {
         return IsmRecord.<V>meta(
-            keyComponents, ByteArrayCoder.of().decode(inStream, context.nested()));
+            keyComponents, ByteArrayCoder.of().decode(inStream));
       } else {
-        return IsmRecord.<V>of(keyComponents, valueCoder.decode(inStream, context.nested()));
+        return IsmRecord.<V>of(keyComponents, valueCoder.decode(inStream));
       }
     }
 
@@ -499,7 +499,7 @@ public void encode(K value, OutputStream outStream, Coder.Context context)
         outStream.write(0);
       } else {
         outStream.write(1);
-        keyCoder.encode(value, outStream, context.nested());
+        keyCoder.encode(value, outStream);
       }
     }
 
@@ -510,7 +510,7 @@ public K decode(InputStream inStream, Coder.Context context)
       if (marker == 0) {
         return (K) getMetadataKey();
       } else if (marker == 1) {
-        return keyCoder.decode(inStream, context.nested());
+        return keyCoder.decode(inStream);
       } else {
         throw new CoderException(String.format("Expected marker but got %s.", marker));
       }
@@ -626,8 +626,8 @@ public void encode(IsmShard value, OutputStream outStream, Coder.Context context
       checkState(value.getIndexOffset() >= 0,
           "%s attempting to be written without index offset.",
           value);
-      VarIntCoder.of().encode(value.getId(), outStream, context.nested());
-      VarLongCoder.of().encode(value.getBlockOffset(), outStream, context.nested());
+      VarIntCoder.of().encode(value.getId(), outStream);
+      VarLongCoder.of().encode(value.getBlockOffset(), outStream);
       VarLongCoder.of().encode(value.getIndexOffset(), outStream, context);
     }
 
@@ -635,8 +635,8 @@ public void encode(IsmShard value, OutputStream outStream, Coder.Context context
     public IsmShard decode(
         InputStream inStream, Coder.Context context) throws CoderException, IOException {
       return IsmShard.of(
-          VarIntCoder.of().decode(inStream, context.nested()),
-          VarLongCoder.of().decode(inStream, context.nested()),
+          VarIntCoder.of().decode(inStream),
+          VarLongCoder.of().decode(inStream),
           VarLongCoder.of().decode(inStream, context));
     }
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
index c836ca5801b8..27f2ec8897c1 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
@@ -207,7 +207,7 @@ private void writeObject(ObjectOutputStream oos) throws IOException {
       oos.writeObject(inCoder);
       try {
         combineFn.getAccumulatorCoder(ctxt.getCoderRegistry(), inCoder)
-            .encode(state, oos, Coder.Context.NESTED);
+            .encode(state, oos);
       } catch (CannotProvideCoderException e) {
         throw new IllegalStateException("Could not determine coder for accumulator", e);
       }
@@ -220,7 +220,7 @@ private void readObject(ObjectInputStream ois) throws IOException, ClassNotFound
       inCoder = (Coder<InputT>) ois.readObject();
       try {
         state = combineFn.getAccumulatorCoder(ctxt.getCoderRegistry(), inCoder)
-            .decode(ois, Coder.Context.NESTED);
+            .decode(ois);
       } catch (CannotProvideCoderException e) {
         throw new IllegalStateException("Could not determine coder for accumulator", e);
       }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
index 97559a9abb4f..e2166cf25d58 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
@@ -51,14 +51,14 @@ private BigDecimalCoder() {}
   public void encode(BigDecimal value, OutputStream outStream, Context context)
       throws IOException, CoderException {
     checkNotNull(value, String.format("cannot encode a null %s", BigDecimal.class.getSimpleName()));
-    VAR_INT_CODER.encode(value.scale(), outStream, context.nested());
+    VAR_INT_CODER.encode(value.scale(), outStream);
     BIG_INT_CODER.encode(value.unscaledValue(), outStream, context);
   }
 
   @Override
   public BigDecimal decode(InputStream inStream, Context context)
       throws IOException, CoderException {
-    int scale = VAR_INT_CODER.decode(inStream, context.nested());
+    int scale = VAR_INT_CODER.decode(inStream);
     BigInteger bigInteger = BIG_INT_CODER.decode(inStream, context);
     return new BigDecimal(bigInteger, scale);
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
index 9994b3f671d7..59d5424d52cc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
@@ -97,7 +97,7 @@ public void encode(
       Collection<T> collection = (Collection<T>) iterable;
       dataOutStream.writeInt(collection.size());
       for (T elem : collection) {
-        elementCoder.encode(elem, dataOutStream, nestedContext);
+        elementCoder.encode(elem, dataOutStream);
       }
     } else {
       // We don't know the size without traversing it so use a fixed size buffer
@@ -108,7 +108,7 @@ public void encode(
           new BufferedElementCountingOutputStream(dataOutStream);
       for (T elem : iterable) {
         countingOutputStream.markElementStart();
-        elementCoder.encode(elem, countingOutputStream, nestedContext);
+        elementCoder.encode(elem, countingOutputStream);
       }
       countingOutputStream.finish();
     }
@@ -125,7 +125,7 @@ public IterableT decode(InputStream inStream, Context context)
     if (size >= 0) {
       List<T> elements = new ArrayList<>(size);
       for (int i = 0; i < size; i++) {
-        elements.add(elementCoder.decode(dataInStream, nestedContext));
+        elements.add(elementCoder.decode(dataInStream));
       }
       return decodeToIterable(elements);
     }
@@ -134,7 +134,7 @@ public IterableT decode(InputStream inStream, Context context)
     // each block of elements.
     long count = VarInt.decodeLong(dataInStream);
     while (count > 0L) {
-      elements.add(elementCoder.decode(dataInStream, nestedContext));
+      elements.add(elementCoder.decode(dataInStream));
       --count;
       if (count == 0L) {
           count = VarInt.decodeLong(dataInStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
index 1df446075f83..0bb53ecbe006 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
@@ -63,14 +63,14 @@ public void encode(KV<K, V> kv, OutputStream outStream, Context context)
     if (kv == null) {
       throw new CoderException("cannot encode a null KV");
     }
-    keyCoder.encode(kv.getKey(), outStream, context.nested());
+    keyCoder.encode(kv.getKey(), outStream);
     valueCoder.encode(kv.getValue(), outStream, context);
   }
 
   @Override
   public KV<K, V> decode(InputStream inStream, Context context)
       throws IOException, CoderException {
-    K key = keyCoder.decode(inStream, context.nested());
+    K key = keyCoder.decode(inStream);
     V value = valueCoder.decode(inStream, context);
     return KV.of(key, value);
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
index 7df9ca96bf38..f20eb93422e0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
@@ -89,12 +89,12 @@ public void encode(
     Iterator<Entry<K, V>> iterator = map.entrySet().iterator();
     Entry<K, V> entry = iterator.next();
     while (iterator.hasNext()) {
-      keyCoder.encode(entry.getKey(), outStream, context.nested());
-      valueCoder.encode(entry.getValue(), outStream, context.nested());
+      keyCoder.encode(entry.getKey(), outStream);
+      valueCoder.encode(entry.getValue(), outStream);
       entry = iterator.next();
     }
 
-    keyCoder.encode(entry.getKey(), outStream, context.nested());
+    keyCoder.encode(entry.getKey(), outStream);
     valueCoder.encode(entry.getValue(), outStream, context);
     // no flush needed as DataOutputStream does not buffer
   }
@@ -110,12 +110,12 @@ public Map<K, V> decode(InputStream inStream, Context context)
 
     Map<K, V> retval = Maps.newHashMapWithExpectedSize(size);
     for (int i = 0; i < size - 1; ++i) {
-      K key = keyCoder.decode(inStream, context.nested());
-      V value = valueCoder.decode(inStream, context.nested());
+      K key = keyCoder.decode(inStream);
+      V value = valueCoder.decode(inStream);
       retval.put(key, value);
     }
 
-    K key = keyCoder.decode(inStream, context.nested());
+    K key = keyCoder.decode(inStream);
     V value = valueCoder.decode(inStream, context);
     retval.put(key, value);
     return retval;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 20fab9b68b62..d8a98cd2e2c6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -952,7 +952,7 @@ public void encode(FileResult value, OutputStream outStream, Context context)
       if (value == null) {
         throw new CoderException("cannot encode a null value");
       }
-      stringCoder.encode(value.getFilename().toString(), outStream, context.nested());
+      stringCoder.encode(value.getFilename().toString(), outStream);
       if (value.getDestinationFilename() == null) {
         stringCoder.encode(null, outStream, context);
       } else {
@@ -963,7 +963,7 @@ public void encode(FileResult value, OutputStream outStream, Context context)
     @Override
     public FileResult decode(InputStream inStream, Context context)
         throws IOException {
-      String filename = stringCoder.decode(inStream, context.nested());
+      String filename = stringCoder.decode(inStream);
       assert filename != null;  // fixes a compiler warning
       @Nullable String destinationFilename = stringCoder.decode(inStream, context);
       return new FileResult(
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
index 37d5a551ccbf..348cc5fca42e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
@@ -682,10 +682,10 @@ public void encode(
         QuantileState<T, ComparatorT> state, OutputStream outStream, Coder.Context context)
         throws CoderException, IOException {
       Coder.Context nestedContext = context.nested();
-      intCoder.encode(state.numQuantiles, outStream, nestedContext);
-      intCoder.encode(state.bufferSize, outStream, nestedContext);
-      elementCoder.encode(state.min, outStream, nestedContext);
-      elementCoder.encode(state.max, outStream, nestedContext);
+      intCoder.encode(state.numQuantiles, outStream);
+      intCoder.encode(state.bufferSize, outStream);
+      elementCoder.encode(state.min, outStream);
+      elementCoder.encode(state.max, outStream);
       elementListCoder.encode(
           state.unbufferedElements, outStream, nestedContext);
       BigEndianIntegerCoder.of().encode(
@@ -699,14 +699,14 @@ public void encode(
     public QuantileState<T, ComparatorT> decode(InputStream inStream, Coder.Context context)
         throws CoderException, IOException {
       Coder.Context nestedContext = context.nested();
-      int numQuantiles = intCoder.decode(inStream, nestedContext);
-      int bufferSize = intCoder.decode(inStream, nestedContext);
-      T min = elementCoder.decode(inStream, nestedContext);
-      T max = elementCoder.decode(inStream, nestedContext);
+      int numQuantiles = intCoder.decode(inStream);
+      int bufferSize = intCoder.decode(inStream);
+      T min = elementCoder.decode(inStream);
+      T max = elementCoder.decode(inStream);
       List<T> unbufferedElements =
-          elementListCoder.decode(inStream, nestedContext);
+          elementListCoder.decode(inStream);
       int numBuffers =
-          BigEndianIntegerCoder.of().decode(inStream, nestedContext);
+          BigEndianIntegerCoder.of().decode(inStream);
       List<QuantileBuffer<T>> buffers = new ArrayList<>(numBuffers);
       for (int i = 0; i < numBuffers; i++) {
         buffers.add(decodeBuffer(inStream, nestedContext));
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
index 0515ed527e50..c45df046f60f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
@@ -543,7 +543,7 @@ public void encode(Object[] value, OutputStream outStream, Context context)
       int lastIndex = codersCount - 1;
       Context nestedContext = context.nested();
       for (int i = 0; i < lastIndex; ++i) {
-        coders.get(i).encode(value[i], outStream, nestedContext);
+        coders.get(i).encode(value[i], outStream);
       }
       coders.get(lastIndex).encode(value[lastIndex], outStream, context);
     }
@@ -558,7 +558,7 @@ public Object[] decode(InputStream inStream, Context context)
       int lastIndex = codersCount - 1;
       Context nestedContext = context.nested();
       for (int i = 0; i < lastIndex; ++i) {
-        ret[i] = coders.get(i).decode(inStream, nestedContext);
+        ret[i] = coders.get(i).decode(inStream);
       }
       ret[lastIndex] = coders.get(lastIndex).decode(inStream, context);
       return ret;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
index a309954982f1..a46a21f43ea5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
@@ -187,7 +187,7 @@ static class CountSumCoder<NumT extends Number> extends AtomicCoder<CountSum<Num
      @Override
      public void encode(CountSum<NumT> value, OutputStream outStream, Coder.Context context)
          throws CoderException, IOException {
-       LONG_CODER.encode(value.count, outStream, context.nested());
+       LONG_CODER.encode(value.count, outStream);
        DOUBLE_CODER.encode(value.sum, outStream, context);
      }
 
@@ -195,7 +195,7 @@ public void encode(CountSum<NumT> value, OutputStream outStream, Coder.Context c
      public CountSum<NumT> decode(InputStream inStream, Coder.Context context)
          throws CoderException, IOException {
        return new CountSum<>(
-           LONG_CODER.decode(inStream, context.nested()),
+           LONG_CODER.decode(inStream),
            DOUBLE_CODER.decode(inStream, context));
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
index e9a3571216ff..bd669efb91ae 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
@@ -253,7 +253,7 @@ public void encode(
       }
       int lastIndex = schema.size() - 1;
       for (int unionTag = 0; unionTag < lastIndex; unionTag++) {
-        tagListCoder(unionTag).encode(value.valueMap.get(unionTag), outStream, context.nested());
+        tagListCoder(unionTag).encode(value.valueMap.get(unionTag), outStream);
       }
       tagListCoder(lastIndex).encode(value.valueMap.get(lastIndex), outStream, context);
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
index 46ece09e0645..cb5a7cf313be 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
@@ -181,14 +181,14 @@ public static IntervalWindowCoder of() {
     @Override
     public void encode(IntervalWindow window, OutputStream outStream, Context context)
         throws IOException, CoderException {
-      instantCoder.encode(window.end, outStream, context.nested());
+      instantCoder.encode(window.end, outStream);
       durationCoder.encode(new Duration(window.start, window.end), outStream, context);
     }
 
     @Override
     public IntervalWindow decode(InputStream inStream, Context context)
         throws IOException, CoderException {
-      Instant end = instantCoder.decode(inStream, context.nested());
+      Instant end = instantCoder.decode(inStream);
       ReadableDuration duration = durationCoder.decode(inStream, context);
       return new IntervalWindow(end.minus(duration), end);
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
index 1b7e335deacf..e3e61cffb1c5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
@@ -643,8 +643,8 @@ public void encode(WindowedValue<T> windowedElem,
       Context nestedContext = context.nested();
       InstantCoder.of().encode(
           windowedElem.getTimestamp(), outStream, nestedContext);
-      windowsCoder.encode(windowedElem.getWindows(), outStream, nestedContext);
-      PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream, nestedContext);
+      windowsCoder.encode(windowedElem.getWindows(), outStream);
+      PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream);
       valueCoder.encode(windowedElem.getValue(), outStream, context);
     }
 
@@ -652,10 +652,10 @@ public void encode(WindowedValue<T> windowedElem,
     public WindowedValue<T> decode(InputStream inStream, Context context)
         throws CoderException, IOException {
       Context nestedContext = context.nested();
-      Instant timestamp = InstantCoder.of().decode(inStream, nestedContext);
+      Instant timestamp = InstantCoder.of().decode(inStream);
       Collection<? extends BoundedWindow> windows =
-          windowsCoder.decode(inStream, nestedContext);
-      PaneInfo pane = PaneInfoCoder.INSTANCE.decode(inStream, nestedContext);
+          windowsCoder.decode(inStream);
+      PaneInfo pane = PaneInfoCoder.INSTANCE.decode(inStream);
       T value = valueCoder.decode(inStream, context);
       return WindowedValue.of(value, timestamp, windows, pane);
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
index c1728857dfbb..89747a7a94a2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
@@ -104,7 +104,7 @@ public void encode(TimestampedValue<T> windowedElem,
                        OutputStream outStream,
                        Context context)
         throws IOException {
-      valueCoder.encode(windowedElem.getValue(), outStream, context.nested());
+      valueCoder.encode(windowedElem.getValue(), outStream);
       InstantCoder.of().encode(
           windowedElem.getTimestamp(), outStream, context);
     }
@@ -112,7 +112,7 @@ public void encode(TimestampedValue<T> windowedElem,
     @Override
     public TimestampedValue<T> decode(InputStream inStream, Context context)
         throws IOException {
-      T value = valueCoder.decode(inStream, context.nested());
+      T value = valueCoder.decode(inStream);
       Instant timestamp = InstantCoder.of().decode(inStream, context);
       return TimestampedValue.of(value, timestamp);
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
index 3ecbaa2072ad..e8a2dfdb516d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
@@ -78,18 +78,18 @@ public static <T> Coder<T> of(
     public void encode(ValueInSingleWindow<T> windowedElem, OutputStream outStream, Context context)
         throws IOException {
       Context nestedContext = context.nested();
-      InstantCoder.of().encode(windowedElem.getTimestamp(), outStream, nestedContext);
-      windowCoder.encode(windowedElem.getWindow(), outStream, nestedContext);
-      PaneInfo.PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream, nestedContext);
+      InstantCoder.of().encode(windowedElem.getTimestamp(), outStream);
+      windowCoder.encode(windowedElem.getWindow(), outStream);
+      PaneInfo.PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream);
       valueCoder.encode(windowedElem.getValue(), outStream, context);
     }
 
     @Override
     public ValueInSingleWindow<T> decode(InputStream inStream, Context context) throws IOException {
       Context nestedContext = context.nested();
-      Instant timestamp = InstantCoder.of().decode(inStream, nestedContext);
-      BoundedWindow window = windowCoder.decode(inStream, nestedContext);
-      PaneInfo pane = PaneInfo.PaneInfoCoder.INSTANCE.decode(inStream, nestedContext);
+      Instant timestamp = InstantCoder.of().decode(inStream);
+      BoundedWindow window = windowCoder.decode(inStream);
+      PaneInfo pane = PaneInfo.PaneInfoCoder.INSTANCE.decode(inStream);
       T value = valueCoder.decode(inStream, context);
       return new AutoValue_ValueInSingleWindow<>(value, timestamp, window, pane);
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
index 3f057e16cf2d..f06317b38571 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
@@ -103,7 +103,7 @@ public List<? extends Coder<?>> getCoderArguments() {
     @Override
     public void encode(ValueWithRecordId<ValueT> value, OutputStream outStream, Context context)
         throws IOException {
-      valueCoder.encode(value.value, outStream, context.nested());
+      valueCoder.encode(value.value, outStream);
       idCoder.encode(value.id, outStream, context);
     }
 
@@ -111,7 +111,7 @@ public void encode(ValueWithRecordId<ValueT> value, OutputStream outStream, Cont
     public ValueWithRecordId<ValueT> decode(InputStream inStream, Context context)
         throws IOException {
       return new ValueWithRecordId<ValueT>(
-          valueCoder.decode(inStream, context.nested()),
+          valueCoder.decode(inStream),
           idCoder.decode(inStream, context));
     }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
index d97eea6b9ed4..adb665286eee 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
@@ -182,15 +182,15 @@ public void testLongStringEncoding() throws Exception {
     // Encode both strings into NESTED form.
     byte[] nestedEncoding;
     try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
-      coder.encode(source, os, Coder.Context.NESTED);
-      coder.encode(source2, os, Coder.Context.NESTED);
+      coder.encode(source, os);
+      coder.encode(source2, os);
       nestedEncoding = os.toByteArray();
     }
 
     // Decode from NESTED form.
     try (ByteArrayInputStream is = new ByteArrayInputStream(nestedEncoding)) {
-      assertEquals(source, coder.decode(is, Coder.Context.NESTED));
-      assertEquals(source2, coder.decode(is, Coder.Context.NESTED));
+      assertEquals(source, coder.decode(is));
+      assertEquals(source2, coder.decode(is));
       assertEquals(0, is.available());
     }
   }
@@ -207,20 +207,20 @@ public void testMixedWithNullsEncoding() throws Exception {
     Coder<String> coder = SerializableCoder.of(String.class);
     byte[] encodedBytes;
     try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
-      coder.encode(null, os, Coder.Context.NESTED);
-      coder.encode("TestValue", os, Coder.Context.NESTED);
-      coder.encode(null, os, Coder.Context.NESTED);
-      coder.encode("TestValue2", os, Coder.Context.NESTED);
-      coder.encode(null, os, Coder.Context.NESTED);
+      coder.encode(null, os);
+      coder.encode("TestValue", os);
+      coder.encode(null, os);
+      coder.encode("TestValue2", os);
+      coder.encode(null, os);
       encodedBytes = os.toByteArray();
     }
 
     try (ByteArrayInputStream is = new ByteArrayInputStream(encodedBytes)) {
-      assertNull(coder.decode(is, Coder.Context.NESTED));
-      assertEquals("TestValue", coder.decode(is,  Coder.Context.NESTED));
-      assertNull(coder.decode(is, Coder.Context.NESTED));
-      assertEquals("TestValue2", coder.decode(is,  Coder.Context.NESTED));
-      assertNull(coder.decode(is, Coder.Context.NESTED));
+      assertNull(coder.decode(is));
+      assertEquals("TestValue", coder.decode(is));
+      assertNull(coder.decode(is));
+      assertEquals("TestValue2", coder.decode(is));
+      assertNull(coder.decode(is));
       assertEquals(0, is.available());
     }
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
index a70af94f3768..e4b016be10bb 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
@@ -878,14 +878,14 @@ private class CountSumCoder extends AtomicCoder<CountSum> {
       @Override
       public void encode(CountSum value, OutputStream outStream,
           Context context) throws CoderException, IOException {
-        LONG_CODER.encode(value.count, outStream, context.nested());
+        LONG_CODER.encode(value.count, outStream);
         DOUBLE_CODER.encode(value.sum, outStream, context);
       }
 
       @Override
       public CountSum decode(InputStream inStream, Coder.Context context)
           throws CoderException, IOException {
-        long count = LONG_CODER.decode(inStream, context.nested());
+        long count = LONG_CODER.decode(inStream);
         double sum = DOUBLE_CODER.decode(inStream, context);
         return new CountSum(count, sum);
       }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
index a4588128604e..7e8a1dd7bfcc 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
@@ -210,14 +210,14 @@ public void encode(
           OutputStream outStream,
           org.apache.beam.sdk.coders.Coder.Context context)
           throws CoderException, IOException {
-        stringCoder.encode(value.myString, outStream, context.nested());
+        stringCoder.encode(value.myString, outStream);
       }
 
       @Override
       public UnserializableRecord decode(
           InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
           throws CoderException, IOException {
-        return new UnserializableRecord(stringCoder.decode(inStream, context.nested()));
+        return new UnserializableRecord(stringCoder.decode(inStream));
       }
     }
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java
index 314b96965fcd..9ae5d688292b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java
@@ -35,7 +35,7 @@ public void testCoderBinaryRepresentation() throws Exception {
     CountingOutputStream out = new CountingOutputStream(ByteStreams.nullOutputStream());
     GlobalWindow.Coder.INSTANCE.encode(GlobalWindow.INSTANCE, out, Context.OUTER);
     assertEquals(0, out.getCount());
-    GlobalWindow.Coder.INSTANCE.encode(GlobalWindow.INSTANCE, out, Context.NESTED);
+    GlobalWindow.Coder.INSTANCE.encode(GlobalWindow.INSTANCE, out);
     assertEquals(0, out.getCount());
   }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java
index 36f7028627ee..894d8a98021b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java
@@ -32,7 +32,6 @@
 import java.util.List;
 import java.util.Random;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.hamcrest.collection.IsIterableContainingInOrder;
 import org.junit.Rule;
 import org.junit.Test;
@@ -180,7 +179,7 @@ private void verifyValues(List<byte[]> expectedValues, InputStream is) throws Ex
     do {
       count = VarInt.decodeLong(is);
       for (int i = 0; i < count; ++i) {
-        values.add(ByteArrayCoder.of().decode(is, Context.NESTED));
+        values.add(ByteArrayCoder.of().decode(is));
       }
     } while(count > 0);
 
@@ -198,7 +197,7 @@ private void verifyValues(List<byte[]> expectedValues, InputStream is) throws Ex
 
     for (byte[] value : values) {
       os.markElementStart();
-      ByteArrayCoder.of().encode(value, os, Context.NESTED);
+      ByteArrayCoder.of().encode(value, os);
     }
     return os;
   }
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
index 18e0d9509da3..37745be9cb9f 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
@@ -110,7 +110,7 @@ public void close() throws Exception {
 
   @Override
   public void accept(WindowedValue<T> t) throws IOException {
-    coder.encode(t, bufferedElements, Context.NESTED);
+    coder.encode(t, bufferedElements);
     counter += 1;
     if (bufferedElements.size() >= bufferLimit) {
       outboundObserver.onNext(convertBufferForTransmission().build());
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
index 24365d8866f8..ece87d26c8e5 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
@@ -71,7 +71,7 @@ public void accept(BeamFnApi.Elements.Data t) {
       InputStream inputStream = t.getData().newInput();
       while (inputStream.available() > 0) {
         counter += 1;
-        WindowedValue<T> value = coder.decode(inputStream, Context.NESTED);
+        WindowedValue<T> value = coder.decode(inputStream);
         consumer.accept(value);
       }
     } catch (Exception e) {
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
index 7cbf8ebe66b0..3f6ece71063b 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
@@ -135,7 +135,7 @@ public void testExperimentConfiguresBufferLimit() throws Exception {
   private static BeamFnApi.Elements messageWithData(byte[] ... datum) throws IOException {
     ByteString.Output output = ByteString.newOutput();
     for (byte[] data : datum) {
-      CODER.encode(valueInGlobalWindow(data), output, Context.NESTED);
+      CODER.encode(valueInGlobalWindow(data), output);
     }
     return BeamFnApi.Elements.newBuilder()
         .addData(BeamFnApi.Elements.Data.newBuilder()
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
index c53f99d49fc1..4b0bf0ce932b 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
@@ -108,7 +108,7 @@ private BeamFnApi.Elements.Data dataWith(String ... values) throws Exception {
             .setName("Test"));
     ByteString.Output output = ByteString.newOutput();
     for (String value : values) {
-      CODER.encode(valueInGlobalWindow(value), output, Context.NESTED);
+      CODER.encode(valueInGlobalWindow(value), output);
     }
     builder.setData(output.toByteString());
     return builder.build();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
index 7aefcfa606c0..c2b62b72357e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
@@ -53,17 +53,18 @@ public List<? extends Coder<?>> getCoderArguments() {
   }
 
   @Override
-  public void encode(ShardedKey<KeyT> key, OutputStream outStream, Context context)
+  public void encode(ShardedKey<KeyT> key, OutputStream outStream)
       throws IOException {
-    keyCoder.encode(key.getKey(), outStream, context.nested());
-    shardNumberCoder.encode(key.getShardNumber(), outStream, context);
+    keyCoder.encode(key.getKey(), outStream);
+    shardNumberCoder.encode(key.getShardNumber(), outStream);
   }
 
   @Override
-  public ShardedKey<KeyT> decode(InputStream inStream, Context context)
+  public ShardedKey<KeyT> decode(InputStream inStream)
       throws IOException {
     return new ShardedKey<>(
-        keyCoder.decode(inStream, context.nested()), shardNumberCoder.decode(inStream, context));
+        keyCoder.decode(inStream),
+        shardNumberCoder.decode(inStream));
   }
 
   @Override
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
index 01bc558214fa..33b9f779ab9d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
@@ -43,14 +43,14 @@ public void encode(TableDestination value, OutputStream outStream, Context conte
     if (value == null) {
       throw new CoderException("cannot encode a null value");
     }
-    tableSpecCoder.encode(value.getTableSpec(), outStream, context.nested());
-    tableDescriptionCoder.encode(value.getTableDescription(), outStream, context);
+    tableSpecCoder.encode(value.getTableSpec(), outStream);
+    tableDescriptionCoder.encode(value.getTableDescription(), outStream);
   }
 
   @Override
-  public TableDestination decode(InputStream inStream, Context context) throws IOException {
-    String tableSpec = tableSpecCoder.decode(inStream, context.nested());
-    String tableDescription = tableDescriptionCoder.decode(inStream, context);
+  public TableDestination decode(InputStream inStream) throws IOException {
+    String tableSpec = tableSpecCoder.decode(inStream);
+    String tableDescription = tableDescriptionCoder.decode(inStream);
     return new TableDestination(tableSpec, tableDescription);
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
index 2b1988a2bda7..8ae75c56581e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
@@ -43,7 +43,7 @@ public void encode(TableRowInfo value, OutputStream outStream, Context context)
     if (value == null) {
       throw new CoderException("cannot encode a null value");
     }
-    tableRowCoder.encode(value.tableRow, outStream, context.nested());
+    tableRowCoder.encode(value.tableRow, outStream);
     idCoder.encode(value.uniqueId, outStream, context);
   }
 
@@ -51,7 +51,7 @@ public void encode(TableRowInfo value, OutputStream outStream, Context context)
   public TableRowInfo decode(InputStream inStream, Context context)
       throws IOException {
     return new TableRowInfo(
-        tableRowCoder.decode(inStream, context.nested()),
+        tableRowCoder.decode(inStream),
         idCoder.decode(inStream, context));
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
index 890979ba2c8d..9e83271a5865 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
@@ -106,16 +106,16 @@ public void encode(Result<DestinationT> value, OutputStream outStream, Context c
       if (value == null) {
         throw new CoderException("cannot encode a null value");
       }
-      stringCoder.encode(value.filename, outStream, context.nested());
-      longCoder.encode(value.fileByteSize, outStream, context.nested());
-      destinationCoder.encode(value.destination, outStream, context.nested());
+      stringCoder.encode(value.filename, outStream);
+      longCoder.encode(value.fileByteSize, outStream);
+      destinationCoder.encode(value.destination, outStream);
     }
 
     @Override
     public Result<DestinationT> decode(InputStream inStream, Context context) throws IOException {
-      String filename = stringCoder.decode(inStream, context.nested());
-      long fileByteSize = longCoder.decode(inStream, context.nested());
-      DestinationT destination = destinationCoder.decode(inStream, context.nested());
+      String filename = stringCoder.decode(inStream);
+      long fileByteSize = longCoder.decode(inStream);
+      DestinationT destination = destinationCoder.decode(inStream);
       return new Result<>(filename, fileByteSize, destination);
     }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
index e061edc50106..5907c9e7a214 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
@@ -47,13 +47,13 @@ public static PubsubMessageWithAttributesCoder of() {
 
   public void encode(PubsubMessage value, OutputStream outStream, Context context)
       throws IOException {
-    PAYLOAD_CODER.encode(value.getPayload(), outStream, context.nested());
+    PAYLOAD_CODER.encode(value.getPayload(), outStream);
     ATTRIBUTES_CODER.encode(value.getAttributeMap(), outStream, context);
   }
 
   @Override
   public PubsubMessage decode(InputStream inStream, Context context) throws IOException {
-    byte[] payload = PAYLOAD_CODER.decode(inStream, context.nested());
+    byte[] payload = PAYLOAD_CODER.decode(inStream);
     Map<String, String> attributes = ATTRIBUTES_CODER.decode(inStream, context);
     return new PubsubMessage(payload, attributes);
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
index 9f04a6c3fe07..ae320c73a05b 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
@@ -110,19 +110,19 @@ private static class OutgoingMessageCoder extends AtomicCoder<OutgoingMessage> {
     public void encode(
         OutgoingMessage value, OutputStream outStream, Context context)
         throws CoderException, IOException {
-      ByteArrayCoder.of().encode(value.elementBytes, outStream, context.nested());
-      ATTRIBUTES_CODER.encode(value.attributes, outStream, context.nested());
-      BigEndianLongCoder.of().encode(value.timestampMsSinceEpoch, outStream, context.nested());
-      RECORD_ID_CODER.encode(value.recordId, outStream, context.nested());
+      ByteArrayCoder.of().encode(value.elementBytes, outStream);
+      ATTRIBUTES_CODER.encode(value.attributes, outStream);
+      BigEndianLongCoder.of().encode(value.timestampMsSinceEpoch, outStream);
+      RECORD_ID_CODER.encode(value.recordId, outStream);
     }
 
     @Override
     public OutgoingMessage decode(
         InputStream inStream, Context context) throws CoderException, IOException {
-      byte[] elementBytes = ByteArrayCoder.of().decode(inStream, context.nested());
-      Map<String, String> attributes = ATTRIBUTES_CODER.decode(inStream, context.nested());
-      long timestampMsSinceEpoch = BigEndianLongCoder.of().decode(inStream, context.nested());
-      @Nullable String recordId = RECORD_ID_CODER.decode(inStream, context.nested());
+      byte[] elementBytes = ByteArrayCoder.of().decode(inStream);
+      Map<String, String> attributes = ATTRIBUTES_CODER.decode(inStream);
+      long timestampMsSinceEpoch = BigEndianLongCoder.of().decode(inStream);
+      @Nullable String recordId = RECORD_ID_CODER.decode(inStream);
       return new OutgoingMessage(elementBytes, attributes, timestampMsSinceEpoch, recordId);
     }
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index c16b8fb70eb2..e53976e5ffb8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -380,7 +380,7 @@ public void encode(PubsubCheckpoint value, OutputStream outStream, Context conte
 
     @Override
     public PubsubCheckpoint decode(InputStream inStream, Context context) throws IOException {
-      String path = SUBSCRIPTION_PATH_CODER.decode(inStream, context.nested());
+      String path = SUBSCRIPTION_PATH_CODER.decode(inStream);
       List<String> notYetReadIds = LIST_CODER.decode(inStream, context);
       return new PubsubCheckpoint(path, null, null, notYetReadIds);
     }
diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
index 2b4503ac5bca..5386a613fcbc 100644
--- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
+++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
@@ -181,8 +181,8 @@ public TestCoder(JAXBCoder<TestType> jaxbCoder) {
     public void encode(TestType value, OutputStream outStream, Context context)
         throws CoderException, IOException {
       Context nestedContext = context.nested();
-      VarIntCoder.of().encode(3, outStream, nestedContext);
-      jaxbCoder.encode(value, outStream, nestedContext);
+      VarIntCoder.of().encode(3, outStream);
+      jaxbCoder.encode(value, outStream);
       VarLongCoder.of().encode(22L, outStream, context);
     }
 
@@ -190,8 +190,8 @@ public void encode(TestType value, OutputStream outStream, Context context)
     public TestType decode(InputStream inStream, Context context)
         throws CoderException, IOException {
       Context nestedContext = context.nested();
-      VarIntCoder.of().decode(inStream, nestedContext);
-      TestType result = jaxbCoder.decode(inStream, nestedContext);
+      VarIntCoder.of().decode(inStream);
+      TestType result = jaxbCoder.decode(inStream);
       VarLongCoder.of().decode(inStream, context);
       return result;
     }

From 996dce37b76b103f104328b7caa65f73a1bcb15a Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Fri, 5 May 2017 16:36:47 -0700
Subject: [PATCH 126/387] Remove contexts from coders where they'll never be
 used.

---
 .../UnboundedReadFromBoundedSource.java       |  4 +--
 .../core/ElementAndRestrictionCoder.java      |  4 +--
 .../beam/runners/core/KeyedWorkItemCoder.java |  4 +--
 .../beam/runners/core/TimerInternals.java     |  4 +--
 .../apache/beam/sdk/coders/DurationCoder.java |  4 +--
 .../apache/beam/sdk/coders/InstantCoder.java  |  4 +--
 .../sdk/transforms/ApproximateQuantiles.java  | 20 +++++------
 .../org/apache/beam/sdk/transforms/Mean.java  |  4 +--
 .../org/apache/beam/sdk/transforms/Top.java   |  4 +--
 .../beam/sdk/transforms/join/CoGbkResult.java | 10 ++----
 .../transforms/windowing/IntervalWindow.java  |  4 +--
 .../beam/sdk/values/TimestampedValue.java     |  4 +--
 .../sdk/io/kinesis/KinesisRecordCoder.java    | 34 +++++++++----------
 13 files changed, 47 insertions(+), 57 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
index ae28e3adbd3e..b74da808da8d 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
@@ -224,7 +224,7 @@ static class CheckpointCoder<T> extends StructuredCoder<Checkpoint<T>> {
       public void encode(Checkpoint<T> value, OutputStream outStream, Context context)
           throws CoderException, IOException {
         elemsCoder.encode(value.residualElements, outStream);
-        sourceCoder.encode(value.residualSource, outStream, context);
+        sourceCoder.encode(value.residualSource, outStream);
       }
 
       @SuppressWarnings("unchecked")
@@ -233,7 +233,7 @@ public Checkpoint<T> decode(InputStream inStream, Context context)
           throws CoderException, IOException {
         return new Checkpoint<>(
             elemsCoder.decode(inStream),
-            sourceCoder.decode(inStream, context));
+            sourceCoder.decode(inStream));
       }
 
       @Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
index 5ddd865a518f..fcb1debb3300 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
@@ -56,14 +56,14 @@ public void encode(
       throw new CoderException("cannot encode a null ElementAndRestriction");
     }
     elementCoder.encode(value.element(), outStream);
-    restrictionCoder.encode(value.restriction(), outStream, context);
+    restrictionCoder.encode(value.restriction(), outStream);
   }
 
   @Override
   public ElementAndRestriction<ElementT, RestrictionT> decode(InputStream inStream, Context context)
       throws IOException {
     ElementT key = elementCoder.decode(inStream);
-    RestrictionT value = restrictionCoder.decode(inStream, context);
+    RestrictionT value = restrictionCoder.decode(inStream);
     return ElementAndRestriction.of(key, value);
   }
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
index ac8a34cfb9b0..0869244c8e4d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
@@ -74,7 +74,7 @@ public void encode(KeyedWorkItem<K, ElemT> value, OutputStream outStream, Coder.
     Coder.Context nestedContext = context.nested();
     keyCoder.encode(value.key(), outStream);
     timersCoder.encode(value.timersIterable(), outStream);
-    elemsCoder.encode(value.elementsIterable(), outStream, context);
+    elemsCoder.encode(value.elementsIterable(), outStream);
   }
 
   @Override
@@ -83,7 +83,7 @@ public KeyedWorkItem<K, ElemT> decode(InputStream inStream, Coder.Context contex
     Coder.Context nestedContext = context.nested();
     K key = keyCoder.decode(inStream);
     Iterable<TimerData> timers = timersCoder.decode(inStream);
-    Iterable<WindowedValue<ElemT>> elems = elemsCoder.decode(inStream, context);
+    Iterable<WindowedValue<ElemT>> elems = elemsCoder.decode(inStream);
     return KeyedWorkItems.workItem(key, timers, elems);
   }
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
index 3607fdd6cb6d..f0a62cd6973a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
@@ -244,7 +244,7 @@ public void encode(TimerData timer, OutputStream outStream, Context context)
       STRING_CODER.encode(timer.getTimerId(), outStream);
       STRING_CODER.encode(timer.getNamespace().stringKey(), outStream);
       INSTANT_CODER.encode(timer.getTimestamp(), outStream);
-      STRING_CODER.encode(timer.getDomain().name(), outStream, context);
+      STRING_CODER.encode(timer.getDomain().name(), outStream);
     }
 
     @Override
@@ -255,7 +255,7 @@ public TimerData decode(InputStream inStream, Context context)
       StateNamespace namespace =
           StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder);
       Instant timestamp = INSTANT_CODER.decode(inStream);
-      TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream, context));
+      TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream));
       return TimerData.of(timerId, namespace, timestamp, domain);
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
index 8b4ae1dde31e..b7db3057c08f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
@@ -59,13 +59,13 @@ public void encode(ReadableDuration value, OutputStream outStream, Context conte
     if (value == null) {
       throw new CoderException("cannot encode a null ReadableDuration");
     }
-    LONG_CODER.encode(toLong(value), outStream, context);
+    LONG_CODER.encode(toLong(value), outStream);
   }
 
   @Override
   public ReadableDuration decode(InputStream inStream, Context context)
       throws CoderException, IOException {
-      return fromLong(LONG_CODER.decode(inStream, context));
+      return fromLong(LONG_CODER.decode(inStream));
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
index 000f40672f10..22b11a3ed08e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
@@ -73,13 +73,13 @@ public void encode(Instant value, OutputStream outStream, Context context)
     if (value == null) {
       throw new CoderException("cannot encode a null Instant");
     }
-    LONG_CODER.encode(ORDER_PRESERVING_CONVERTER.convert(value), outStream, context);
+    LONG_CODER.encode(ORDER_PRESERVING_CONVERTER.convert(value), outStream);
   }
 
   @Override
   public Instant decode(InputStream inStream, Context context)
       throws CoderException, IOException {
-    return ORDER_PRESERVING_CONVERTER.reverse().convert(LONG_CODER.decode(inStream, context));
+    return ORDER_PRESERVING_CONVERTER.reverse().convert(LONG_CODER.decode(inStream));
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
index 348cc5fca42e..9b9d3f8a106f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
@@ -681,24 +681,22 @@ public QuantileStateCoder(ComparatorT compareFn, Coder<T> elementCoder) {
     public void encode(
         QuantileState<T, ComparatorT> state, OutputStream outStream, Coder.Context context)
         throws CoderException, IOException {
-      Coder.Context nestedContext = context.nested();
       intCoder.encode(state.numQuantiles, outStream);
       intCoder.encode(state.bufferSize, outStream);
       elementCoder.encode(state.min, outStream);
       elementCoder.encode(state.max, outStream);
       elementListCoder.encode(
-          state.unbufferedElements, outStream, nestedContext);
+          state.unbufferedElements, outStream);
       BigEndianIntegerCoder.of().encode(
-          state.buffers.size(), outStream, nestedContext);
+          state.buffers.size(), outStream);
       for (QuantileBuffer<T> buffer : state.buffers) {
-        encodeBuffer(buffer, outStream, nestedContext);
+        encodeBuffer(buffer, outStream);
       }
     }
 
     @Override
     public QuantileState<T, ComparatorT> decode(InputStream inStream, Coder.Context context)
         throws CoderException, IOException {
-      Coder.Context nestedContext = context.nested();
       int numQuantiles = intCoder.decode(inStream);
       int bufferSize = intCoder.decode(inStream);
       T min = elementCoder.decode(inStream);
@@ -709,29 +707,27 @@ public QuantileState<T, ComparatorT> decode(InputStream inStream, Coder.Context
           BigEndianIntegerCoder.of().decode(inStream);
       List<QuantileBuffer<T>> buffers = new ArrayList<>(numBuffers);
       for (int i = 0; i < numBuffers; i++) {
-        buffers.add(decodeBuffer(inStream, nestedContext));
+        buffers.add(decodeBuffer(inStream));
       }
       return new QuantileState<T, ComparatorT>(
           compareFn, numQuantiles, min, max, numBuffers, bufferSize, unbufferedElements, buffers);
     }
 
-    private void encodeBuffer(
-        QuantileBuffer<T> buffer, OutputStream outStream, Coder.Context context)
+    private void encodeBuffer(QuantileBuffer<T> buffer, OutputStream outStream)
         throws CoderException, IOException {
       DataOutputStream outData = new DataOutputStream(outStream);
       outData.writeInt(buffer.level);
       outData.writeLong(buffer.weight);
-      elementListCoder.encode(buffer.elements, outStream, context);
+      elementListCoder.encode(buffer.elements, outStream);
     }
 
-    private QuantileBuffer<T> decodeBuffer(
-        InputStream inStream, Coder.Context context)
+    private QuantileBuffer<T> decodeBuffer(InputStream inStream)
         throws IOException, CoderException {
       DataInputStream inData = new DataInputStream(inStream);
       return new QuantileBuffer<>(
           inData.readInt(),
           inData.readLong(),
-          elementListCoder.decode(inStream, context));
+          elementListCoder.decode(inStream));
     }
 
     /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
index a46a21f43ea5..c8e0d954d174 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
@@ -188,7 +188,7 @@ static class CountSumCoder<NumT extends Number> extends AtomicCoder<CountSum<Num
      public void encode(CountSum<NumT> value, OutputStream outStream, Coder.Context context)
          throws CoderException, IOException {
        LONG_CODER.encode(value.count, outStream);
-       DOUBLE_CODER.encode(value.sum, outStream, context);
+       DOUBLE_CODER.encode(value.sum, outStream);
      }
 
      @Override
@@ -196,7 +196,7 @@ public CountSum<NumT> decode(InputStream inStream, Coder.Context context)
          throws CoderException, IOException {
        return new CountSum<>(
            LONG_CODER.decode(inStream),
-           DOUBLE_CODER.decode(inStream, context));
+           DOUBLE_CODER.decode(inStream));
     }
 
     @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
index c0381a71ee5d..7aec66730288 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
@@ -541,13 +541,13 @@ public BoundedHeapCoder(int maximumSize, ComparatorT compareFn, Coder<T> element
     public void encode(
         BoundedHeap<T, ComparatorT> value, OutputStream outStream, Context context)
         throws CoderException, IOException {
-      listCoder.encode(value.asList(), outStream, context);
+      listCoder.encode(value.asList(), outStream);
     }
 
     @Override
     public BoundedHeap<T, ComparatorT> decode(InputStream inStream, Coder.Context context)
         throws CoderException, IOException {
-      return new BoundedHeap<>(maximumSize, compareFn, listCoder.decode(inStream, context));
+      return new BoundedHeap<>(maximumSize, compareFn, listCoder.decode(inStream));
     }
 
     @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
index bd669efb91ae..66033255184d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
@@ -251,11 +251,9 @@ public void encode(
       if (schema.size() == 0) {
         return;
       }
-      int lastIndex = schema.size() - 1;
-      for (int unionTag = 0; unionTag < lastIndex; unionTag++) {
+      for (int unionTag = 0; unionTag < schema.size(); unionTag++) {
         tagListCoder(unionTag).encode(value.valueMap.get(unionTag), outStream);
       }
-      tagListCoder(lastIndex).encode(value.valueMap.get(lastIndex), outStream, context);
     }
 
     @Override
@@ -266,12 +264,10 @@ public CoGbkResult decode(
       if (schema.size() == 0) {
         return new CoGbkResult(schema, ImmutableList.<Iterable<?>>of());
       }
-      int lastIndex = schema.size() - 1;
       List<Iterable<?>> valueMap = Lists.newArrayListWithExpectedSize(schema.size());
-      for (int unionTag = 0; unionTag < lastIndex; unionTag++) {
-        valueMap.add(tagListCoder(unionTag).decode(inStream, context.nested()));
+      for (int unionTag = 0; unionTag < schema.size(); unionTag++) {
+        valueMap.add(tagListCoder(unionTag).decode(inStream, Coder.Context.NESTED));
       }
-      valueMap.add(tagListCoder(lastIndex).decode(inStream, context));
       return new CoGbkResult(schema, valueMap);
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
index cb5a7cf313be..318dc4ca97bb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
@@ -182,14 +182,14 @@ public static IntervalWindowCoder of() {
     public void encode(IntervalWindow window, OutputStream outStream, Context context)
         throws IOException, CoderException {
       instantCoder.encode(window.end, outStream);
-      durationCoder.encode(new Duration(window.start, window.end), outStream, context);
+      durationCoder.encode(new Duration(window.start, window.end), outStream);
     }
 
     @Override
     public IntervalWindow decode(InputStream inStream, Context context)
         throws IOException, CoderException {
       Instant end = instantCoder.decode(inStream);
-      ReadableDuration duration = durationCoder.decode(inStream, context);
+      ReadableDuration duration = durationCoder.decode(inStream);
       return new IntervalWindow(end.minus(duration), end);
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
index 89747a7a94a2..95a3152d874e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
@@ -106,14 +106,14 @@ public void encode(TimestampedValue<T> windowedElem,
         throws IOException {
       valueCoder.encode(windowedElem.getValue(), outStream);
       InstantCoder.of().encode(
-          windowedElem.getTimestamp(), outStream, context);
+          windowedElem.getTimestamp(), outStream);
     }
 
     @Override
     public TimestampedValue<T> decode(InputStream inStream, Context context)
         throws IOException {
       T value = valueCoder.decode(inStream);
-      Instant timestamp = InstantCoder.of().decode(inStream, context);
+      Instant timestamp = InstantCoder.of().decode(inStream);
       return TimestampedValue.of(value, timestamp);
     }
 
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
index 77fe127bcba6..c6a017461ad4 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
@@ -45,28 +45,26 @@ public static KinesisRecordCoder of() {
     @Override
     public void encode(KinesisRecord value, OutputStream outStream, Context context) throws
             IOException {
-        Context nested = context.nested();
-        BYTE_ARRAY_CODER.encode(value.getData().array(), outStream, nested);
-        STRING_CODER.encode(value.getSequenceNumber(), outStream, nested);
-        STRING_CODER.encode(value.getPartitionKey(), outStream, nested);
-        INSTANT_CODER.encode(value.getApproximateArrivalTimestamp(), outStream, nested);
-        VAR_LONG_CODER.encode(value.getSubSequenceNumber(), outStream, nested);
-        INSTANT_CODER.encode(value.getReadTime(), outStream, nested);
-        STRING_CODER.encode(value.getStreamName(), outStream, nested);
-        STRING_CODER.encode(value.getShardId(), outStream, context);
+        BYTE_ARRAY_CODER.encode(value.getData().array(), outStream);
+        STRING_CODER.encode(value.getSequenceNumber(), outStream);
+        STRING_CODER.encode(value.getPartitionKey(), outStream);
+        INSTANT_CODER.encode(value.getApproximateArrivalTimestamp(), outStream);
+        VAR_LONG_CODER.encode(value.getSubSequenceNumber(), outStream);
+        INSTANT_CODER.encode(value.getReadTime(), outStream);
+        STRING_CODER.encode(value.getStreamName(), outStream);
+        STRING_CODER.encode(value.getShardId(), outStream);
     }
 
     @Override
     public KinesisRecord decode(InputStream inStream, Context context) throws IOException {
-        Context nested = context.nested();
-        ByteBuffer data = ByteBuffer.wrap(BYTE_ARRAY_CODER.decode(inStream, nested));
-        String sequenceNumber = STRING_CODER.decode(inStream, nested);
-        String partitionKey = STRING_CODER.decode(inStream, nested);
-        Instant approximateArrivalTimestamp = INSTANT_CODER.decode(inStream, nested);
-        long subSequenceNumber = VAR_LONG_CODER.decode(inStream, nested);
-        Instant readTimestamp = INSTANT_CODER.decode(inStream, nested);
-        String streamName = STRING_CODER.decode(inStream, nested);
-        String shardId = STRING_CODER.decode(inStream, context);
+        ByteBuffer data = ByteBuffer.wrap(BYTE_ARRAY_CODER.decode(inStream));
+        String sequenceNumber = STRING_CODER.decode(inStream);
+        String partitionKey = STRING_CODER.decode(inStream);
+        Instant approximateArrivalTimestamp = INSTANT_CODER.decode(inStream);
+        long subSequenceNumber = VAR_LONG_CODER.decode(inStream);
+        Instant readTimestamp = INSTANT_CODER.decode(inStream);
+        String streamName = STRING_CODER.decode(inStream);
+        String shardId = STRING_CODER.decode(inStream);
         return new KinesisRecord(data, sequenceNumber, subSequenceNumber, partitionKey,
                 approximateArrivalTimestamp, readTimestamp, streamName, shardId
         );

From b7f3341ed4fc073a834a55773bcb4a2c7f821c52 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Fri, 5 May 2017 17:24:02 -0700
Subject: [PATCH 127/387] automated context removal or redirection

---
 .../translation/utils/ApexStreamTuple.java    | 11 ++++++++
 .../UnboundedReadFromBoundedSource.java       |  4 +--
 .../runners/core/construction/CodersTest.java |  4 +--
 .../core/ElementAndRestrictionCoder.java      |  4 +--
 .../beam/runners/core/KeyedWorkItemCoder.java |  6 ++--
 .../beam/runners/core/TimerInternals.java     |  6 ++--
 .../direct/CloningBundleFactoryTest.java      | 20 ++++++-------
 .../UnboundedReadEvaluatorFactoryTest.java    |  5 ++--
 .../SingletonKeyedWorkItemCoder.java          | 11 ++++++++
 .../runners/dataflow/BatchViewOverrides.java  | 11 ++++++++
 .../runners/dataflow/internal/IsmFormat.java  | 28 +++++++++++++------
 .../dataflow/util/RandomAccessData.java       | 11 ++++++++
 .../org/apache/beam/sdk/coders/AvroCoder.java |  4 +--
 .../beam/sdk/coders/BigDecimalCoder.java      | 11 ++++++++
 .../sdk/coders/BigEndianIntegerCoder.java     |  4 +--
 .../beam/sdk/coders/BigEndianLongCoder.java   |  4 +--
 .../beam/sdk/coders/BigIntegerCoder.java      | 11 ++++++++
 .../apache/beam/sdk/coders/BitSetCoder.java   | 11 ++++++++
 .../beam/sdk/coders/ByteArrayCoder.java       | 11 ++++++++
 .../org/apache/beam/sdk/coders/ByteCoder.java |  4 +--
 .../apache/beam/sdk/coders/DelegateCoder.java | 11 ++++++++
 .../apache/beam/sdk/coders/DoubleCoder.java   |  4 +--
 .../apache/beam/sdk/coders/DurationCoder.java |  4 +--
 .../apache/beam/sdk/coders/InstantCoder.java  |  4 +--
 .../beam/sdk/coders/IterableLikeCoder.java    |  6 ++--
 .../org/apache/beam/sdk/coders/KvCoder.java   | 11 ++++++++
 .../beam/sdk/coders/LengthPrefixCoder.java    |  4 +--
 .../org/apache/beam/sdk/coders/MapCoder.java  | 11 ++++++++
 .../apache/beam/sdk/coders/NullableCoder.java | 11 ++++++++
 .../beam/sdk/coders/SerializableCoder.java    |  4 +--
 .../beam/sdk/coders/StringDelegateCoder.java  | 11 ++++++++
 .../beam/sdk/coders/StringUtf8Coder.java      | 11 ++++++++
 .../beam/sdk/coders/TextualIntegerCoder.java  | 11 ++++++++
 .../apache/beam/sdk/coders/VarIntCoder.java   |  4 +--
 .../apache/beam/sdk/coders/VarLongCoder.java  |  4 +--
 .../org/apache/beam/sdk/coders/VoidCoder.java |  4 +--
 .../org/apache/beam/sdk/io/FileBasedSink.java | 11 ++++++++
 .../sdk/transforms/ApproximateQuantiles.java  |  4 +--
 .../apache/beam/sdk/transforms/Combine.java   | 22 +++++++++++++++
 .../beam/sdk/transforms/CombineFns.java       | 13 +++++++--
 .../org/apache/beam/sdk/transforms/Count.java |  4 +--
 .../org/apache/beam/sdk/transforms/Mean.java  |  4 +--
 .../org/apache/beam/sdk/transforms/Top.java   |  4 +--
 .../beam/sdk/transforms/join/CoGbkResult.java |  6 ++--
 .../beam/sdk/transforms/join/UnionCoder.java  | 11 ++++++++
 .../transforms/windowing/GlobalWindow.java    |  4 +--
 .../transforms/windowing/IntervalWindow.java  |  4 +--
 .../sdk/transforms/windowing/PaneInfo.java    |  4 +--
 .../org/apache/beam/sdk/util/BitSetCoder.java | 11 ++++++++
 .../apache/beam/sdk/util/WindowedValue.java   | 24 ++++++++++++++--
 .../beam/sdk/values/TimestampedValue.java     |  5 ++--
 .../beam/sdk/values/ValueInSingleWindow.java  | 13 +++++++--
 .../beam/sdk/values/ValueWithRecordId.java    | 11 ++++++++
 .../beam/sdk/coders/CoderRegistryTest.java    |  8 +++---
 .../beam/sdk/coders/CustomCoderTest.java      |  4 +--
 .../beam/sdk/coders/NullableCoderTest.java    | 11 ++++++++
 .../beam/sdk/coders/StructuredCoderTest.java  | 12 ++++----
 .../apache/beam/sdk/testing/PAssertTest.java  |  4 +--
 .../sdk/testing/SerializableMatchersTest.java |  4 +--
 .../beam/sdk/testing/WindowSupplierTest.java  |  4 +--
 .../beam/sdk/transforms/CombineFnsTest.java   | 11 ++++++++
 .../beam/sdk/transforms/CombineTest.java      | 22 +++++++++++++++
 .../beam/sdk/transforms/CreateTest.java       |  9 +++---
 .../beam/sdk/transforms/GroupByKeyTest.java   |  4 +--
 .../apache/beam/sdk/transforms/ParDoTest.java | 15 ++++++++--
 .../apache/beam/sdk/transforms/ViewTest.java  | 11 ++++++++
 .../transforms/reflect/DoFnInvokersTest.java  |  8 +++---
 .../apache/beam/sdk/util/CoderUtilsTest.java  |  4 +--
 .../beam/sdk/util/SerializableUtilsTest.java  |  4 +--
 .../extensions/protobuf/ByteStringCoder.java  | 11 ++++++++
 .../sdk/extensions/protobuf/ProtoCoder.java   | 11 ++++++++
 .../gcp/bigquery/TableDestinationCoder.java   |  2 +-
 .../io/gcp/bigquery/TableRowInfoCoder.java    | 11 ++++++++
 .../io/gcp/bigquery/TableRowJsonCoder.java    | 11 ++++++++
 .../io/gcp/bigquery/WriteBundlesToFiles.java  |  4 +--
 .../pubsub/PubsubMessagePayloadOnlyCoder.java | 11 ++++++++
 .../PubsubMessageWithAttributesCoder.java     | 11 ++++++++
 .../io/gcp/pubsub/PubsubUnboundedSink.java    |  4 +--
 .../io/gcp/pubsub/PubsubUnboundedSource.java  | 11 ++++++++
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java   | 11 ++++++++
 .../beam/sdk/io/hadoop/WritableCoder.java     |  4 +--
 .../beam/sdk/io/hbase/HBaseMutationCoder.java |  6 ++--
 .../beam/sdk/io/hbase/HBaseResultCoder.java   |  4 +--
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java |  4 +--
 .../beam/sdk/io/kafka/KafkaRecordCoder.java   | 11 ++++++++
 .../sdk/io/kinesis/KinesisRecordCoder.java    |  4 +--
 .../org/apache/beam/sdk/io/xml/JAXBCoder.java | 25 ++++++++++-------
 .../apache/beam/sdk/io/xml/JAXBCoderTest.java | 13 +++++++--
 88 files changed, 598 insertions(+), 157 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java
index 4aa6ee82017e..1d402eb450fc 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStreamTuple.java
@@ -161,6 +161,12 @@ protected ApexStreamTupleCoder(Coder<T> valueCoder) {
       this.valueCoder = checkNotNull(valueCoder);
     }
 
+    @Override
+    public void encode(ApexStreamTuple<T> value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(ApexStreamTuple<T> value, OutputStream outStream, Context context)
         throws CoderException, IOException {
@@ -174,6 +180,11 @@ public void encode(ApexStreamTuple<T> value, OutputStream outStream, Context con
       }
     }
 
+    @Override
+    public ApexStreamTuple<T> decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public ApexStreamTuple<T> decode(InputStream inStream, Context context)
         throws CoderException, IOException {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
index b74da808da8d..24eb38479268 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java
@@ -221,7 +221,7 @@ static class CheckpointCoder<T> extends StructuredCoder<Checkpoint<T>> {
       }
 
       @Override
-      public void encode(Checkpoint<T> value, OutputStream outStream, Context context)
+      public void encode(Checkpoint<T> value, OutputStream outStream)
           throws CoderException, IOException {
         elemsCoder.encode(value.residualElements, outStream);
         sourceCoder.encode(value.residualSource, outStream);
@@ -229,7 +229,7 @@ public void encode(Checkpoint<T> value, OutputStream outStream, Context context)
 
       @SuppressWarnings("unchecked")
       @Override
-      public Checkpoint<T> decode(InputStream inStream, Context context)
+      public Checkpoint<T> decode(InputStream inStream)
           throws CoderException, IOException {
         return new Checkpoint<>(
             elemsCoder.decode(inStream),
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
index 765723c9bb3d..42fba7cda508 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
@@ -151,11 +151,11 @@ static class Record implements Serializable {}
 
     private static class RecordCoder extends AtomicCoder<Record> {
       @Override
-      public void encode(Record value, OutputStream outStream, Context context)
+      public void encode(Record value, OutputStream outStream)
           throws CoderException, IOException {}
 
       @Override
-      public Record decode(InputStream inStream, Context context)
+      public Record decode(InputStream inStream)
           throws CoderException, IOException {
         return new Record();
       }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
index fcb1debb3300..4440b856318b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
@@ -50,7 +50,7 @@ private ElementAndRestrictionCoder(
 
   @Override
   public void encode(
-      ElementAndRestriction<ElementT, RestrictionT> value, OutputStream outStream, Context context)
+      ElementAndRestriction<ElementT, RestrictionT> value, OutputStream outStream)
       throws IOException {
     if (value == null) {
       throw new CoderException("cannot encode a null ElementAndRestriction");
@@ -60,7 +60,7 @@ public void encode(
   }
 
   @Override
-  public ElementAndRestriction<ElementT, RestrictionT> decode(InputStream inStream, Context context)
+  public ElementAndRestriction<ElementT, RestrictionT> decode(InputStream inStream)
       throws IOException {
     ElementT key = elementCoder.decode(inStream);
     RestrictionT value = restrictionCoder.decode(inStream);
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
index 0869244c8e4d..b1cb1a61eb5b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java
@@ -69,18 +69,16 @@ public Coder<ElemT> getElementCoder() {
   }
 
   @Override
-  public void encode(KeyedWorkItem<K, ElemT> value, OutputStream outStream, Coder.Context context)
+  public void encode(KeyedWorkItem<K, ElemT> value, OutputStream outStream)
       throws CoderException, IOException {
-    Coder.Context nestedContext = context.nested();
     keyCoder.encode(value.key(), outStream);
     timersCoder.encode(value.timersIterable(), outStream);
     elemsCoder.encode(value.elementsIterable(), outStream);
   }
 
   @Override
-  public KeyedWorkItem<K, ElemT> decode(InputStream inStream, Coder.Context context)
+  public KeyedWorkItem<K, ElemT> decode(InputStream inStream)
       throws CoderException, IOException {
-    Coder.Context nestedContext = context.nested();
     K key = keyCoder.decode(inStream);
     Iterable<TimerData> timers = timersCoder.decode(inStream);
     Iterable<WindowedValue<ElemT>> elems = elemsCoder.decode(inStream);
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
index f0a62cd6973a..f4a12d089116 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java
@@ -238,9 +238,8 @@ private TimerDataCoder(Coder<? extends BoundedWindow> windowCoder) {
     }
 
     @Override
-    public void encode(TimerData timer, OutputStream outStream, Context context)
+    public void encode(TimerData timer, OutputStream outStream)
         throws CoderException, IOException {
-      Context nestedContext = context.nested();
       STRING_CODER.encode(timer.getTimerId(), outStream);
       STRING_CODER.encode(timer.getNamespace().stringKey(), outStream);
       INSTANT_CODER.encode(timer.getTimestamp(), outStream);
@@ -248,9 +247,8 @@ public void encode(TimerData timer, OutputStream outStream, Context context)
     }
 
     @Override
-    public TimerData decode(InputStream inStream, Context context)
+    public TimerData decode(InputStream inStream)
         throws CoderException, IOException {
-      Context nestedContext = context.nested();
       String timerId = STRING_CODER.decode(inStream);
       StateNamespace namespace =
           StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder);
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java
index 33d171e50449..5bc807748ad3 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java
@@ -178,15 +178,14 @@ static class RecordNoEncodeCoder extends AtomicCoder<Record> {
     @Override
     public void encode(
         Record value,
-        OutputStream outStream,
-        org.apache.beam.sdk.coders.Coder.Context context)
+        OutputStream outStream)
         throws IOException {
       throw new CoderException("Encode not allowed");
     }
 
     @Override
     public Record decode(
-        InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+        InputStream inStream)
         throws IOException {
       return null;
     }
@@ -196,13 +195,12 @@ static class RecordNoDecodeCoder extends AtomicCoder<Record> {
     @Override
     public void encode(
         Record value,
-        OutputStream outStream,
-        org.apache.beam.sdk.coders.Coder.Context context)
+        OutputStream outStream)
         throws IOException {}
 
     @Override
     public Record decode(
-        InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+        InputStream inStream)
         throws IOException {
       throw new CoderException("Decode not allowed");
     }
@@ -212,13 +210,12 @@ private static class RecordStructuralValueCoder extends AtomicCoder<Record> {
     @Override
     public void encode(
         Record value,
-        OutputStream outStream,
-        org.apache.beam.sdk.coders.Coder.Context context)
+        OutputStream outStream)
         throws CoderException, IOException {}
 
     @Override
     public Record decode(
-        InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+        InputStream inStream)
         throws CoderException, IOException {
       return new Record() {
         @Override
@@ -244,13 +241,12 @@ private static class RecordNotConsistentWithEqualsStructuralValueCoder
     @Override
     public void encode(
         Record value,
-        OutputStream outStream,
-        org.apache.beam.sdk.coders.Coder.Context context)
+        OutputStream outStream)
         throws CoderException, IOException {}
 
     @Override
     public Record decode(
-        InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+        InputStream inStream)
         throws CoderException, IOException {
       return new Record() {
         @Override
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index b9ba7f49df79..2a01db55ae1e 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -590,15 +590,14 @@ public static class Coder extends AtomicCoder<TestCheckpointMark> {
       @Override
       public void encode(
           TestCheckpointMark value,
-          OutputStream outStream,
-          org.apache.beam.sdk.coders.Coder.Context context)
+          OutputStream outStream)
           throws IOException {
         VarInt.encode(value.index, outStream);
       }
 
       @Override
       public TestCheckpointMark decode(
-          InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+          InputStream inStream)
           throws IOException {
         TestCheckpointMark decoded = new TestCheckpointMark(VarInt.decodeInt(inStream));
         decoded.decoded = true;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
index d7bae7e7b3db..b62fc16fbe6c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
@@ -65,6 +65,12 @@ public Coder<ElemT> getElementCoder() {
     return elemCoder;
   }
 
+  @Override
+  public void encode(SingletonKeyedWorkItem<K, ElemT> value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(SingletonKeyedWorkItem<K, ElemT> value,
                      OutputStream outStream,
@@ -74,6 +80,11 @@ public void encode(SingletonKeyedWorkItem<K, ElemT> value,
     valueCoder.encode(value.value, outStream, context);
   }
 
+  @Override
+  public SingletonKeyedWorkItem<K, ElemT> decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public SingletonKeyedWorkItem<K, ElemT> decode(InputStream inStream, Context context)
       throws CoderException, IOException {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index 0e60fa09dba9..34609dfecce3 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -1350,6 +1350,12 @@ public static <K, V1, V2> TransformedMapCoder<K, V1, V2> of(
       return new TransformedMapCoder<>(transformCoder, originalMapCoder);
     }
 
+    @Override
+    public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(outStream, outStream, Coder.Context.NESTED);
+    }
+
     @Override
     public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream,
         Coder.Context context) throws CoderException, IOException {
@@ -1357,6 +1363,11 @@ public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream,
       originalMapCoder.encode(value.originalMap, outStream, context);
     }
 
+    @Override
+    public TransformedMap<K, V1, V2> decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Coder.Context.NESTED);
+    }
+
     @Override
     public TransformedMap<K, V1, V2> decode(
         InputStream inStream, Coder.Context context) throws CoderException, IOException {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
index 0f0cd4d940d7..8cfae816d217 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
@@ -231,8 +231,7 @@ public Coder<V> getValueCoder() {
     }
 
     @Override
-    public void encode(IsmRecord<V> value, OutputStream outStream,
-        Coder.Context context) throws CoderException, IOException {
+    public void encode(IsmRecord<V> value, OutputStream outStream) throws CoderException, IOException {
       if (value.getKeyComponents().size() != keyComponentCoders.size()) {
         throw new CoderException(String.format(
             "Expected %s key component(s) but received key component(s) %s.",
@@ -249,7 +248,7 @@ public void encode(IsmRecord<V> value, OutputStream outStream,
     }
 
     @Override
-    public IsmRecord<V> decode(InputStream inStream, Coder.Context context)
+    public IsmRecord<V> decode(InputStream inStream)
         throws CoderException, IOException {
       List<Object> keyComponents = new ArrayList<>(keyComponentCoders.size());
       for (Coder<?> keyCoder : keyComponentCoders) {
@@ -493,7 +492,7 @@ public Coder<K> getKeyCoder() {
     }
 
     @Override
-    public void encode(K value, OutputStream outStream, Coder.Context context)
+    public void encode(K value, OutputStream outStream)
         throws CoderException, IOException {
       if (value == METADATA_KEY) {
         outStream.write(0);
@@ -504,7 +503,7 @@ public void encode(K value, OutputStream outStream, Coder.Context context)
     }
 
     @Override
-    public K decode(InputStream inStream, Coder.Context context)
+    public K decode(InputStream inStream)
         throws CoderException, IOException {
       int marker = inStream.read();
       if (marker == 0) {
@@ -620,6 +619,12 @@ public static IsmShardCoder of() {
 
     private IsmShardCoder() {}
 
+    @Override
+    public void encode(IsmShard value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Coder.Context.NESTED);
+    }
+
     @Override
     public void encode(IsmShard value, OutputStream outStream, Coder.Context context)
         throws CoderException, IOException {
@@ -631,6 +636,11 @@ public void encode(IsmShard value, OutputStream outStream, Coder.Context context
       VarLongCoder.of().encode(value.getIndexOffset(), outStream, context);
     }
 
+    @Override
+    public IsmShard decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Coder.Context.NESTED);
+    }
+
     @Override
     public IsmShard decode(
         InputStream inStream, Coder.Context context) throws CoderException, IOException {
@@ -683,14 +693,14 @@ public static KeyPrefixCoder of() {
     }
 
     @Override
-    public void encode(KeyPrefix value, OutputStream outStream, Coder.Context context)
+    public void encode(KeyPrefix value, OutputStream outStream)
         throws CoderException, IOException {
       VarInt.encode(value.getSharedKeySize(), outStream);
       VarInt.encode(value.getUnsharedKeySize(), outStream);
     }
 
     @Override
-    public KeyPrefix decode(InputStream inStream, Coder.Context context)
+    public KeyPrefix decode(InputStream inStream)
         throws CoderException, IOException {
       return KeyPrefix.of(VarInt.decodeInt(inStream), VarInt.decodeInt(inStream));
     }
@@ -755,7 +765,7 @@ public static FooterCoder of() {
     }
 
     @Override
-    public void encode(Footer value, OutputStream outStream, Coder.Context context)
+    public void encode(Footer value, OutputStream outStream)
         throws CoderException, IOException {
       DataOutputStream dataOut = new DataOutputStream(outStream);
       dataOut.writeLong(value.getIndexPosition());
@@ -765,7 +775,7 @@ public void encode(Footer value, OutputStream outStream, Coder.Context context)
     }
 
     @Override
-    public Footer decode(InputStream inStream, Coder.Context context)
+    public Footer decode(InputStream inStream)
         throws CoderException, IOException {
       DataInputStream dataIn = new DataInputStream(inStream);
       Footer footer = Footer.of(dataIn.readLong(), dataIn.readLong(), dataIn.readLong());
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
index f36bd78511e2..5ea9f07ff6a9 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
@@ -62,6 +62,12 @@ public static RandomAccessDataCoder of() {
       return INSTANCE;
     }
 
+    @Override
+    public void encode(RandomAccessData value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Coder.Context.NESTED);
+    }
+
     @Override
     public void encode(RandomAccessData value, OutputStream outStream, Coder.Context context)
         throws CoderException, IOException {
@@ -74,6 +80,11 @@ public void encode(RandomAccessData value, OutputStream outStream, Coder.Context
       value.writeTo(outStream, 0, value.size);
     }
 
+    @Override
+    public RandomAccessData decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Coder.Context.NESTED);
+    }
+
     @Override
     public RandomAccessData decode(InputStream inStream, Coder.Context context)
         throws CoderException, IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
index f82c065f488e..bba669de2302 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
@@ -300,7 +300,7 @@ public Class<T> getType() {
   }
 
   @Override
-  public void encode(T value, OutputStream outStream, Context context) throws IOException {
+  public void encode(T value, OutputStream outStream) throws IOException {
     // Get a BinaryEncoder instance from the ThreadLocal cache and attempt to reuse it.
     BinaryEncoder encoderInstance = ENCODER_FACTORY.directBinaryEncoder(outStream, encoder.get());
     // Save the potentially-new instance for reuse later.
@@ -310,7 +310,7 @@ public void encode(T value, OutputStream outStream, Context context) throws IOEx
   }
 
   @Override
-  public T decode(InputStream inStream, Context context) throws IOException {
+  public T decode(InputStream inStream) throws IOException {
     // Get a BinaryDecoder instance from the ThreadLocal cache and attempt to reuse it.
     BinaryDecoder decoderInstance = DECODER_FACTORY.directBinaryDecoder(inStream, decoder.get());
     // Save the potentially-new instance for later.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
index e2166cf25d58..e890d118648d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java
@@ -47,6 +47,12 @@ public static BigDecimalCoder of() {
 
   private BigDecimalCoder() {}
 
+  @Override
+  public void encode(BigDecimal value, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(BigDecimal value, OutputStream outStream, Context context)
       throws IOException, CoderException {
@@ -55,6 +61,11 @@ public void encode(BigDecimal value, OutputStream outStream, Context context)
     BIG_INT_CODER.encode(value.unscaledValue(), outStream, context);
   }
 
+  @Override
+  public BigDecimal decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public BigDecimal decode(InputStream inStream, Context context)
       throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
index a61f09951173..efb1e4b7a8e9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianIntegerCoder.java
@@ -43,7 +43,7 @@ public static BigEndianIntegerCoder of() {
   private BigEndianIntegerCoder() {}
 
   @Override
-  public void encode(Integer value, OutputStream outStream, Context context)
+  public void encode(Integer value, OutputStream outStream)
       throws IOException, CoderException {
     if (value == null) {
       throw new CoderException("cannot encode a null Integer");
@@ -52,7 +52,7 @@ public void encode(Integer value, OutputStream outStream, Context context)
   }
 
   @Override
-  public Integer decode(InputStream inStream, Context context)
+  public Integer decode(InputStream inStream)
       throws IOException, CoderException {
     try {
       return new DataInputStream(inStream).readInt();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
index 868160aad1e0..ab85e17e759a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigEndianLongCoder.java
@@ -43,7 +43,7 @@ public static BigEndianLongCoder of() {
   private BigEndianLongCoder() {}
 
   @Override
-  public void encode(Long value, OutputStream outStream, Context context)
+  public void encode(Long value, OutputStream outStream)
       throws IOException, CoderException {
     if (value == null) {
       throw new CoderException("cannot encode a null Long");
@@ -52,7 +52,7 @@ public void encode(Long value, OutputStream outStream, Context context)
   }
 
   @Override
-  public Long decode(InputStream inStream, Context context)
+  public Long decode(InputStream inStream)
       throws IOException, CoderException {
     try {
       return new DataInputStream(inStream).readLong();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
index 3b038afefbe3..d54accfb0fbb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java
@@ -41,6 +41,12 @@ public static BigIntegerCoder of() {
 
   private BigIntegerCoder() {}
 
+  @Override
+  public void encode(BigInteger value, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(BigInteger value, OutputStream outStream, Context context)
       throws IOException, CoderException {
@@ -48,6 +54,11 @@ public void encode(BigInteger value, OutputStream outStream, Context context)
     BYTE_ARRAY_CODER.encode(value.toByteArray(), outStream, context);
   }
 
+  @Override
+  public BigInteger decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public BigInteger decode(InputStream inStream, Context context)
       throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java
index f49776b28a5f..811501726237 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitSetCoder.java
@@ -35,6 +35,12 @@ public static BitSetCoder of() {
     return INSTANCE;
   }
 
+  @Override
+  public void encode(BitSet value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(BitSet value, OutputStream outStream, Context context)
       throws CoderException, IOException {
@@ -44,6 +50,11 @@ public void encode(BitSet value, OutputStream outStream, Context context)
     BYTE_ARRAY_CODER.encodeAndOwn(value.toByteArray(), outStream, context);
   }
 
+  @Override
+  public BitSet decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public BitSet decode(InputStream inStream, Context context)
       throws CoderException, IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
index c9393a180046..3b3838823814 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
@@ -51,6 +51,12 @@ public static ByteArrayCoder of() {
 
   private ByteArrayCoder() {}
 
+  @Override
+  public void encode(byte[] value, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(byte[] value, OutputStream outStream, Context context)
       throws IOException, CoderException {
@@ -85,6 +91,11 @@ public void encodeAndOwn(byte[] value, OutputStream outStream, Context context)
     }
   }
 
+  @Override
+  public byte[] decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public byte[] decode(InputStream inStream, Context context)
       throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
index 7f449d6ae895..2d23a642f27b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteCoder.java
@@ -41,7 +41,7 @@ public static ByteCoder of() {
   private ByteCoder() {}
 
   @Override
-  public void encode(Byte value, OutputStream outStream, Context context)
+  public void encode(Byte value, OutputStream outStream)
       throws IOException, CoderException {
     if (value == null) {
       throw new CoderException("cannot encode a null Byte");
@@ -50,7 +50,7 @@ public void encode(Byte value, OutputStream outStream, Context context)
   }
 
   @Override
-  public Byte decode(InputStream inStream, Context context)
+  public Byte decode(InputStream inStream)
       throws IOException, CoderException {
     try {
       // value will be between 0-255, -1 for EOF
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java
index 86077eb120ce..f51b1564b36f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java
@@ -65,12 +65,23 @@ public static <T, IntermediateT> DelegateCoder<T, IntermediateT> of(
     return new DelegateCoder<T, IntermediateT>(coder, toFn, fromFn, typeDescriptor);
   }
 
+  @Override
+  public void encode(T value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(T value, OutputStream outStream, Context context)
       throws CoderException, IOException {
     coder.encode(applyAndWrapExceptions(toFn, value), outStream, context);
   }
 
+  @Override
+  public T decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public T decode(InputStream inStream, Context context) throws CoderException, IOException {
     return applyAndWrapExceptions(fromFn, coder.decode(inStream, context));
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
index 8eff6ba9b6fb..deb18f22401c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DoubleCoder.java
@@ -43,7 +43,7 @@ public static DoubleCoder of() {
   private DoubleCoder() {}
 
   @Override
-  public void encode(Double value, OutputStream outStream, Context context)
+  public void encode(Double value, OutputStream outStream)
       throws IOException, CoderException {
     if (value == null) {
       throw new CoderException("cannot encode a null Double");
@@ -52,7 +52,7 @@ public void encode(Double value, OutputStream outStream, Context context)
   }
 
   @Override
-  public Double decode(InputStream inStream, Context context)
+  public Double decode(InputStream inStream)
       throws IOException, CoderException {
     try {
       return new DataInputStream(inStream).readDouble();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
index b7db3057c08f..90de26fffa0a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DurationCoder.java
@@ -54,7 +54,7 @@ private ReadableDuration fromLong(Long decoded) {
   }
 
   @Override
-  public void encode(ReadableDuration value, OutputStream outStream, Context context)
+  public void encode(ReadableDuration value, OutputStream outStream)
       throws CoderException, IOException {
     if (value == null) {
       throw new CoderException("cannot encode a null ReadableDuration");
@@ -63,7 +63,7 @@ public void encode(ReadableDuration value, OutputStream outStream, Context conte
   }
 
   @Override
-  public ReadableDuration decode(InputStream inStream, Context context)
+  public ReadableDuration decode(InputStream inStream)
       throws CoderException, IOException {
       return fromLong(LONG_CODER.decode(inStream));
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
index 22b11a3ed08e..648493ea2ddc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/InstantCoder.java
@@ -68,7 +68,7 @@ protected Instant doBackward(Long shiftedMillis) {
   }
 
   @Override
-  public void encode(Instant value, OutputStream outStream, Context context)
+  public void encode(Instant value, OutputStream outStream)
       throws CoderException, IOException {
     if (value == null) {
       throw new CoderException("cannot encode a null Instant");
@@ -77,7 +77,7 @@ public void encode(Instant value, OutputStream outStream, Context context)
   }
 
   @Override
-  public Instant decode(InputStream inStream, Context context)
+  public Instant decode(InputStream inStream)
       throws CoderException, IOException {
     return ORDER_PRESERVING_CONVERTER.reverse().convert(LONG_CODER.decode(inStream));
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
index 59d5424d52cc..248c26caed06 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java
@@ -84,12 +84,11 @@ protected IterableLikeCoder(Coder<T> elementCoder, String  iterableName) {
 
   @Override
   public void encode(
-      IterableT iterable, OutputStream outStream, Context context)
+      IterableT iterable, OutputStream outStream)
       throws IOException, CoderException  {
     if (iterable == null) {
       throw new CoderException("cannot encode a null " + iterableName);
     }
-    Context nestedContext = context.nested();
     DataOutputStream dataOutStream = new DataOutputStream(outStream);
     if (iterable instanceof Collection) {
       // We can know the size of the Iterable.  Use an encoding with a
@@ -117,9 +116,8 @@ public void encode(
   }
 
   @Override
-  public IterableT decode(InputStream inStream, Context context)
+  public IterableT decode(InputStream inStream)
       throws IOException, CoderException {
-    Context nestedContext = context.nested();
     DataInputStream dataInStream = new DataInputStream(inStream);
     int size = dataInStream.readInt();
     if (size >= 0) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
index 0bb53ecbe006..9c0188671a4a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/KvCoder.java
@@ -57,6 +57,12 @@ private KvCoder(Coder<K> keyCoder, Coder<V> valueCoder) {
     this.valueCoder = valueCoder;
   }
 
+  @Override
+  public void encode(KV<K, V> kv, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(kv, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(KV<K, V> kv, OutputStream outStream, Context context)
       throws IOException, CoderException  {
@@ -67,6 +73,11 @@ public void encode(KV<K, V> kv, OutputStream outStream, Context context)
     valueCoder.encode(kv.getValue(), outStream, context);
   }
 
+  @Override
+  public KV<K, V> decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public KV<K, V> decode(InputStream inStream, Context context)
       throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
index 7dd2a32098a7..b24f66d3a083 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java
@@ -53,7 +53,7 @@ private LengthPrefixCoder(Coder<T> valueCoder) {
   }
 
   @Override
-  public void encode(T value, OutputStream outStream, Context context)
+  public void encode(T value, OutputStream outStream)
       throws CoderException, IOException {
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     valueCoder.encode(value, bos, Context.OUTER);
@@ -62,7 +62,7 @@ public void encode(T value, OutputStream outStream, Context context)
   }
 
   @Override
-  public T decode(InputStream inStream, Context context) throws CoderException, IOException {
+  public T decode(InputStream inStream) throws CoderException, IOException {
     long size = VarInt.decodeLong(inStream);
     return valueCoder.decode(ByteStreams.limit(inStream, size), Context.OUTER);
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
index f20eb93422e0..d8b3f1cdabe8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java
@@ -68,6 +68,12 @@ private MapCoder(Coder<K> keyCoder, Coder<V> valueCoder) {
     this.valueCoder = valueCoder;
   }
 
+  @Override
+  public void encode(Map<K, V> map, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(map, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(
       Map<K, V> map,
@@ -99,6 +105,11 @@ public void encode(
     // no flush needed as DataOutputStream does not buffer
   }
 
+  @Override
+  public Map<K, V> decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public Map<K, V> decode(InputStream inStream, Context context)
       throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
index e46591e4c799..64229e8edc50 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java
@@ -60,6 +60,12 @@ public Coder<T> getValueCoder() {
     return valueCoder;
   }
 
+  @Override
+  public void encode(@Nullable T value, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(@Nullable T value, OutputStream outStream, Context context)
       throws IOException, CoderException  {
@@ -71,6 +77,11 @@ public void encode(@Nullable T value, OutputStream outStream, Context context)
     }
   }
 
+  @Override
+  public T decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   @Nullable
   public T decode(InputStream inStream, Context context) throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
index e3b2959fd98b..9aa8493e5e71 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
@@ -119,7 +119,7 @@ public Class<T> getRecordType() {
   }
 
   @Override
-  public void encode(T value, OutputStream outStream, Context context)
+  public void encode(T value, OutputStream outStream)
       throws IOException, CoderException {
     try {
       ObjectOutputStream oos = new ObjectOutputStream(outStream);
@@ -131,7 +131,7 @@ public void encode(T value, OutputStream outStream, Context context)
   }
 
   @Override
-  public T decode(InputStream inStream, Context context)
+  public T decode(InputStream inStream)
       throws IOException, CoderException {
     try {
       ObjectInputStream ois = new ObjectInputStream(inStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java
index 1f4538fc1eff..216129157a5b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringDelegateCoder.java
@@ -99,12 +99,23 @@ public int hashCode() {
     return this.clazz.hashCode();
   }
 
+  @Override
+  public void encode(T value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(T value, OutputStream outStream, Context context)
       throws CoderException, IOException {
     delegateCoder.encode(value, outStream, context);
   }
 
+  @Override
+  public T decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public T decode(InputStream inStream, Context context) throws CoderException, IOException {
     return delegateCoder.decode(inStream, context);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
index 44856e8a045f..3bbc983f6551 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java
@@ -66,6 +66,12 @@ private static String readString(DataInputStream dis) throws IOException {
 
   private StringUtf8Coder() {}
 
+  @Override
+  public void encode(String value, OutputStream outStream)
+      throws IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(String value, OutputStream outStream, Context context)
       throws IOException {
@@ -84,6 +90,11 @@ public void encode(String value, OutputStream outStream, Context context)
     }
   }
 
+  @Override
+  public String decode(InputStream inStream) throws IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public String decode(InputStream inStream, Context context)
       throws IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
index 718811c9061b..6078fa3c594f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TextualIntegerCoder.java
@@ -38,6 +38,12 @@ public static TextualIntegerCoder of() {
 
   protected TextualIntegerCoder() {}
 
+  @Override
+  public void encode(Integer value, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(Integer value, OutputStream outStream, Context context)
       throws IOException, CoderException {
@@ -48,6 +54,11 @@ public void encode(Integer value, OutputStream outStream, Context context)
     StringUtf8Coder.of().encode(textualValue, outStream, context);
   }
 
+  @Override
+  public Integer decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public Integer decode(InputStream inStream, Context context)
       throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
index bda66bbeb40d..3a9abe77ca99 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarIntCoder.java
@@ -44,7 +44,7 @@ public static VarIntCoder of() {
   private VarIntCoder() {}
 
   @Override
-  public void encode(Integer value, OutputStream outStream, Context context)
+  public void encode(Integer value, OutputStream outStream)
       throws IOException, CoderException {
     if (value == null) {
       throw new CoderException("cannot encode a null Integer");
@@ -53,7 +53,7 @@ public void encode(Integer value, OutputStream outStream, Context context)
   }
 
   @Override
-  public Integer decode(InputStream inStream, Context context)
+  public Integer decode(InputStream inStream)
       throws IOException, CoderException {
     try {
       return VarInt.decodeInt(inStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
index bf651c37bb6f..37ad8f62a872 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VarLongCoder.java
@@ -45,7 +45,7 @@ public static VarLongCoder of() {
   private VarLongCoder() {}
 
   @Override
-  public void encode(Long value, OutputStream outStream, Context context)
+  public void encode(Long value, OutputStream outStream)
       throws IOException, CoderException {
     if (value == null) {
       throw new CoderException("cannot encode a null Long");
@@ -54,7 +54,7 @@ public void encode(Long value, OutputStream outStream, Context context)
   }
 
   @Override
-  public Long decode(InputStream inStream, Context context)
+  public Long decode(InputStream inStream)
       throws IOException, CoderException {
     try {
       return VarInt.decodeLong(inStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
index 4467faa84f62..3e1ff7f688f7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/VoidCoder.java
@@ -38,12 +38,12 @@ public static VoidCoder of() {
   private VoidCoder() {}
 
   @Override
-  public void encode(Void value, OutputStream outStream, Context context) {
+  public void encode(Void value, OutputStream outStream) {
     // Nothing to write!
   }
 
   @Override
-  public Void decode(InputStream inStream, Context context) {
+  public Void decode(InputStream inStream) {
     // Nothing to read!
     return null;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index d8a98cd2e2c6..3620c22737ec 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -946,6 +946,12 @@ public static FileResultCoder of() {
       return INSTANCE;
     }
 
+    @Override
+    public void encode(FileResult value, OutputStream outStream)
+        throws IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(FileResult value, OutputStream outStream, Context context)
         throws IOException {
@@ -960,6 +966,11 @@ public void encode(FileResult value, OutputStream outStream, Context context)
       }
     }
 
+    @Override
+    public FileResult decode(InputStream inStream) throws IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public FileResult decode(InputStream inStream, Context context)
         throws IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
index 9b9d3f8a106f..d12d193edc00 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
@@ -679,7 +679,7 @@ public QuantileStateCoder(ComparatorT compareFn, Coder<T> elementCoder) {
 
     @Override
     public void encode(
-        QuantileState<T, ComparatorT> state, OutputStream outStream, Coder.Context context)
+        QuantileState<T, ComparatorT> state, OutputStream outStream)
         throws CoderException, IOException {
       intCoder.encode(state.numQuantiles, outStream);
       intCoder.encode(state.bufferSize, outStream);
@@ -695,7 +695,7 @@ public void encode(
     }
 
     @Override
-    public QuantileState<T, ComparatorT> decode(InputStream inStream, Coder.Context context)
+    public QuantileState<T, ComparatorT> decode(InputStream inStream)
         throws CoderException, IOException {
       int numQuantiles = intCoder.decode(inStream);
       int bufferSize = intCoder.decode(inStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index b9cdbd5f407f..7e435647ed15 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -531,6 +531,12 @@ public HolderCoder(Coder<V> valueCoder) {
       this.valueCoder = valueCoder;
     }
 
+    @Override
+    public void encode(Holder<V> accumulator, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(accumulator, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(Holder<V> accumulator, OutputStream outStream, Context context)
         throws CoderException, IOException {
@@ -542,6 +548,11 @@ public void encode(Holder<V> accumulator, OutputStream outStream, Context contex
       }
     }
 
+    @Override
+    public Holder<V> decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public Holder<V> decode(InputStream inStream, Context context)
         throws CoderException, IOException {
@@ -1970,6 +1981,12 @@ public InputOrAccumCoder(Coder<InputT> inputCoder, Coder<AccumT> accumCoder) {
           this.accumCoder = accumCoder;
         }
 
+        @Override
+        public void encode(InputOrAccum<InputT, AccumT> value, OutputStream outStream)
+            throws CoderException, IOException {
+          encode(value, outStream, Coder.Context.NESTED);
+        }
+
         @Override
         public void encode(
             InputOrAccum<InputT, AccumT> value, OutputStream outStream, Coder.Context context)
@@ -1983,6 +2000,11 @@ public void encode(
           }
         }
 
+        @Override
+        public InputOrAccum<InputT, AccumT> decode(InputStream inStream) throws CoderException, IOException {
+          return decode(inStream, Coder.Context.NESTED);
+        }
+
         @Override
         public InputOrAccum<InputT, AccumT> decode(InputStream inStream, Coder.Context context)
             throws CoderException, IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
index c45df046f60f..c61978367d9e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
@@ -533,6 +533,12 @@ public ComposedAccumulatorCoder(List<Coder<Object>> coders) {
       this.codersCount  = coders.size();
     }
 
+    @Override
+    public void encode(Object[] value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(Object[] value, OutputStream outStream, Context context)
         throws CoderException, IOException {
@@ -541,13 +547,17 @@ public void encode(Object[] value, OutputStream outStream, Context context)
         return;
       }
       int lastIndex = codersCount - 1;
-      Context nestedContext = context.nested();
       for (int i = 0; i < lastIndex; ++i) {
         coders.get(i).encode(value[i], outStream);
       }
       coders.get(lastIndex).encode(value[lastIndex], outStream, context);
     }
 
+    @Override
+    public Object[] decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public Object[] decode(InputStream inStream, Context context)
         throws CoderException, IOException {
@@ -556,7 +566,6 @@ public Object[] decode(InputStream inStream, Context context)
         return ret;
       }
       int lastIndex = codersCount - 1;
-      Context nestedContext = context.nested();
       for (int i = 0; i < lastIndex; ++i) {
         ret[i] = coders.get(i).decode(inStream);
       }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
index 497d62b2f4a1..b405dd1aae30 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
@@ -169,13 +169,13 @@ public Coder<long[]> getAccumulatorCoder(CoderRegistry registry,
                                              Coder<T> inputCoder) {
       return new AtomicCoder<long[]>() {
         @Override
-        public void encode(long[] value, OutputStream outStream, Context context)
+        public void encode(long[] value, OutputStream outStream)
             throws IOException {
           VarInt.encode(value[0], outStream);
         }
 
         @Override
-        public long[] decode(InputStream inStream, Context context)
+        public long[] decode(InputStream inStream)
             throws IOException, CoderException {
           try {
             return new long[] {VarInt.decodeLong(inStream)};
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
index c8e0d954d174..8932b03e5271 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java
@@ -185,14 +185,14 @@ static class CountSumCoder<NumT extends Number> extends AtomicCoder<CountSum<Num
      private static final Coder<Double> DOUBLE_CODER = DoubleCoder.of();
 
      @Override
-     public void encode(CountSum<NumT> value, OutputStream outStream, Coder.Context context)
+     public void encode(CountSum<NumT> value, OutputStream outStream)
          throws CoderException, IOException {
        LONG_CODER.encode(value.count, outStream);
        DOUBLE_CODER.encode(value.sum, outStream);
      }
 
      @Override
-     public CountSum<NumT> decode(InputStream inStream, Coder.Context context)
+     public CountSum<NumT> decode(InputStream inStream)
          throws CoderException, IOException {
        return new CountSum<>(
            LONG_CODER.decode(inStream),
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
index 7aec66730288..dd8bc5f6f659 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
@@ -539,13 +539,13 @@ public BoundedHeapCoder(int maximumSize, ComparatorT compareFn, Coder<T> element
 
     @Override
     public void encode(
-        BoundedHeap<T, ComparatorT> value, OutputStream outStream, Context context)
+        BoundedHeap<T, ComparatorT> value, OutputStream outStream)
         throws CoderException, IOException {
       listCoder.encode(value.asList(), outStream);
     }
 
     @Override
-    public BoundedHeap<T, ComparatorT> decode(InputStream inStream, Coder.Context context)
+    public BoundedHeap<T, ComparatorT> decode(InputStream inStream)
         throws CoderException, IOException {
       return new BoundedHeap<>(maximumSize, compareFn, listCoder.decode(inStream));
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
index 66033255184d..d42de829c789 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
@@ -242,8 +242,7 @@ public UnionCoder getUnionCoder() {
     @SuppressWarnings("unchecked")
     public void encode(
         CoGbkResult value,
-        OutputStream outStream,
-        Context context) throws CoderException,
+        OutputStream outStream) throws CoderException,
         IOException {
       if (!schema.equals(value.getSchema())) {
         throw new CoderException("input schema does not match coder schema");
@@ -258,8 +257,7 @@ public void encode(
 
     @Override
     public CoGbkResult decode(
-        InputStream inStream,
-        Context context)
+        InputStream inStream)
         throws CoderException, IOException {
       if (schema.size() == 0) {
         return new CoGbkResult(schema, ImmutableList.<Iterable<?>>of());
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
index 3194a37f4966..66959d34aa47 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
@@ -54,6 +54,12 @@ private int getIndexForEncoding(RawUnionValue union) {
     return index;
   }
 
+  @Override
+  public void encode(RawUnionValue union, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(union, outStream, Context.NESTED);
+  }
+
   @SuppressWarnings("unchecked")
   @Override
   public void encode(
@@ -73,6 +79,11 @@ public void encode(
         context);
   }
 
+  @Override
+  public RawUnionValue decode(InputStream inStream) throws IOException, CoderException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public RawUnionValue decode(InputStream inStream, Context context)
       throws IOException, CoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
index 0bfb8750dcdc..078cbeed087f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
@@ -67,10 +67,10 @@ public static class Coder extends StructuredCoder<GlobalWindow> {
     public static final Coder INSTANCE = new Coder();
 
     @Override
-    public void encode(GlobalWindow window, OutputStream outStream, Context context) {}
+    public void encode(GlobalWindow window, OutputStream outStream) {}
 
     @Override
-    public GlobalWindow decode(InputStream inStream, Context context) {
+    public GlobalWindow decode(InputStream inStream) {
       return GlobalWindow.INSTANCE;
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
index 318dc4ca97bb..f25a2086fa17 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IntervalWindow.java
@@ -179,14 +179,14 @@ public static IntervalWindowCoder of() {
     }
 
     @Override
-    public void encode(IntervalWindow window, OutputStream outStream, Context context)
+    public void encode(IntervalWindow window, OutputStream outStream)
         throws IOException, CoderException {
       instantCoder.encode(window.end, outStream);
       durationCoder.encode(new Duration(window.start, window.end), outStream);
     }
 
     @Override
-    public IntervalWindow decode(InputStream inStream, Context context)
+    public IntervalWindow decode(InputStream inStream)
         throws IOException, CoderException {
       Instant end = instantCoder.decode(inStream);
       ReadableDuration duration = durationCoder.decode(inStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
index 79ce2f5bf24a..75df220ab873 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
@@ -347,7 +347,7 @@ public static PaneInfoCoder of() {
     private PaneInfoCoder() {}
 
     @Override
-    public void encode(PaneInfo value, final OutputStream outStream, Coder.Context context)
+    public void encode(PaneInfo value, final OutputStream outStream)
         throws CoderException, IOException {
       Encoding encoding = chooseEncoding(value);
       switch (chooseEncoding(value)) {
@@ -369,7 +369,7 @@ public void encode(PaneInfo value, final OutputStream outStream, Coder.Context c
     }
 
     @Override
-    public PaneInfo decode(final InputStream inStream, Coder.Context context)
+    public PaneInfo decode(final InputStream inStream)
         throws CoderException, IOException {
       byte keyAndTag = (byte) inStream.read();
       PaneInfo base = BYTE_TO_PANE_INFO.get((byte) (keyAndTag & 0x0F));
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
index a0896f53cfd7..b20206569bbd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BitSetCoder.java
@@ -42,12 +42,23 @@ public static BitSetCoder of() {
     return INSTANCE;
   }
 
+  @Override
+  public void encode(BitSet value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(BitSet value, OutputStream outStream, Context context)
       throws CoderException, IOException {
     BYTE_ARRAY_CODER.encodeAndOwn(value.toByteArray(), outStream, context);
   }
 
+  @Override
+  public BitSet decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public BitSet decode(InputStream inStream, Context context)
       throws CoderException, IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
index e3e61cffb1c5..963886bee9d6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
@@ -635,12 +635,17 @@ public <NewT> WindowedValueCoder<NewT> withValueCoder(Coder<NewT> valueCoder) {
       return new FullWindowedValueCoder<>(valueCoder, windowCoder);
     }
 
+    @Override
+    public void encode(WindowedValue<T> windowedElem, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(windowedElem, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(WindowedValue<T> windowedElem,
                        OutputStream outStream,
                        Context context)
         throws CoderException, IOException {
-      Context nestedContext = context.nested();
       InstantCoder.of().encode(
           windowedElem.getTimestamp(), outStream, nestedContext);
       windowsCoder.encode(windowedElem.getWindows(), outStream);
@@ -648,10 +653,14 @@ public void encode(WindowedValue<T> windowedElem,
       valueCoder.encode(windowedElem.getValue(), outStream, context);
     }
 
+    @Override
+    public WindowedValue<T> decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public WindowedValue<T> decode(InputStream inStream, Context context)
         throws CoderException, IOException {
-      Context nestedContext = context.nested();
       Instant timestamp = InstantCoder.of().decode(inStream);
       Collection<? extends BoundedWindow> windows =
           windowsCoder.decode(inStream);
@@ -709,12 +718,23 @@ public <NewT> WindowedValueCoder<NewT> withValueCoder(Coder<NewT> valueCoder) {
       return new ValueOnlyWindowedValueCoder<>(valueCoder);
     }
 
+    @Override
+    public void encode(WindowedValue<T> windowedElem, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(windowedElem, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(WindowedValue<T> windowedElem, OutputStream outStream, Context context)
         throws CoderException, IOException {
       valueCoder.encode(windowedElem.getValue(), outStream, context);
     }
 
+    @Override
+    public WindowedValue<T> decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public WindowedValue<T> decode(InputStream inStream, Context context)
         throws CoderException, IOException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
index 95a3152d874e..a4c8b3fa4b4b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java
@@ -101,8 +101,7 @@ public static <T> TimestampedValueCoder<T> of(Coder<T> valueCoder) {
 
     @Override
     public void encode(TimestampedValue<T> windowedElem,
-                       OutputStream outStream,
-                       Context context)
+                       OutputStream outStream)
         throws IOException {
       valueCoder.encode(windowedElem.getValue(), outStream);
       InstantCoder.of().encode(
@@ -110,7 +109,7 @@ public void encode(TimestampedValue<T> windowedElem,
     }
 
     @Override
-    public TimestampedValue<T> decode(InputStream inStream, Context context)
+    public TimestampedValue<T> decode(InputStream inStream)
         throws IOException {
       T value = valueCoder.decode(inStream);
       Instant timestamp = InstantCoder.of().decode(inStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
index e8a2dfdb516d..24c3c3835e41 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java
@@ -74,19 +74,28 @@ public static <T> Coder<T> of(
       this.windowCoder = (org.apache.beam.sdk.coders.Coder) windowCoder;
     }
 
+    @Override
+    public void encode(ValueInSingleWindow<T> windowedElem, OutputStream outStream)
+        throws IOException {
+      encode(windowedElem, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(ValueInSingleWindow<T> windowedElem, OutputStream outStream, Context context)
         throws IOException {
-      Context nestedContext = context.nested();
       InstantCoder.of().encode(windowedElem.getTimestamp(), outStream);
       windowCoder.encode(windowedElem.getWindow(), outStream);
       PaneInfo.PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream);
       valueCoder.encode(windowedElem.getValue(), outStream, context);
     }
 
+    @Override
+    public ValueInSingleWindow<T> decode(InputStream inStream) throws IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public ValueInSingleWindow<T> decode(InputStream inStream, Context context) throws IOException {
-      Context nestedContext = context.nested();
       Instant timestamp = InstantCoder.of().decode(inStream);
       BoundedWindow window = windowCoder.decode(inStream);
       PaneInfo pane = PaneInfo.PaneInfoCoder.INSTANCE.decode(inStream);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
index f06317b38571..96a5f1db91c9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java
@@ -100,6 +100,12 @@ public List<? extends Coder<?>> getCoderArguments() {
       return Arrays.asList(valueCoder);
     }
 
+    @Override
+    public void encode(ValueWithRecordId<ValueT> value, OutputStream outStream)
+        throws IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(ValueWithRecordId<ValueT> value, OutputStream outStream, Context context)
         throws IOException {
@@ -107,6 +113,11 @@ public void encode(ValueWithRecordId<ValueT> value, OutputStream outStream, Cont
       idCoder.encode(value.id, outStream, context);
     }
 
+    @Override
+    public ValueWithRecordId<ValueT> decode(InputStream inStream) throws IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public ValueWithRecordId<ValueT> decode(InputStream inStream, Context context)
         throws IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
index 7ca7fb993717..c883ca021366 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
@@ -81,12 +81,12 @@ public void testRegisterInstantiatedCoder() throws Exception {
   @SuppressWarnings("rawtypes") // this class exists to fail a test because of its rawtypes
   private class MyListCoder extends AtomicCoder<List> {
     @Override
-    public void encode(List value, OutputStream outStream, Context context)
+    public void encode(List value, OutputStream outStream)
         throws CoderException, IOException {
     }
 
     @Override
-    public List decode(InputStream inStream, Context context)
+    public List decode(InputStream inStream)
         throws CoderException, IOException {
       return Collections.emptyList();
     }
@@ -375,12 +375,12 @@ public static MyValueCoder of() {
     }
 
     @Override
-    public void encode(MyValue value, OutputStream outStream, Context context)
+    public void encode(MyValue value, OutputStream outStream)
         throws CoderException, IOException {
     }
 
     @Override
-    public MyValue decode(InputStream inStream, Context context)
+    public MyValue decode(InputStream inStream)
         throws CoderException, IOException {
       return new MyValue();
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CustomCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CustomCoderTest.java
index dfd4ea252e7f..13a72619f946 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CustomCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CustomCoderTest.java
@@ -48,13 +48,13 @@ public MyCustomCoder(String key) {
     }
 
     @Override
-    public void encode(KV<String, Long> kv, OutputStream out, Context context)
+    public void encode(KV<String, Long> kv, OutputStream out)
             throws IOException {
       new DataOutputStream(out).writeLong(kv.getValue());
     }
 
     @Override
-    public KV<String, Long> decode(InputStream inStream, Context context)
+    public KV<String, Long> decode(InputStream inStream)
         throws IOException {
       return KV.of(key, new DataInputStream(inStream).readLong());
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
index d6d7de8b5779..9fb0b822a7a7 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java
@@ -166,6 +166,12 @@ public void testEncodedTypeDescriptor() throws Exception {
   }
 
   private static class EntireStreamExpectingCoder extends AtomicCoder<String> {
+    @Override
+    public void encode(String value, OutputStream outStream)
+        throws IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(
         String value, OutputStream outStream, Context context) throws IOException {
@@ -173,6 +179,11 @@ public void encode(
       StringUtf8Coder.of().encode(value, outStream, context);
     }
 
+    @Override
+    public String decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public String decode(InputStream inStream, Context context)
         throws CoderException, IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java
index af2c94e1c68b..7aa2080cf101 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java
@@ -47,7 +47,7 @@ private static class NullBooleanCoder extends StructuredCoder<Boolean> {
     private static final long serialVersionUID = 0L;
 
     @Override
-    public void encode(@Nullable Boolean value, OutputStream outStream, Context context)
+    public void encode(@Nullable Boolean value, OutputStream outStream)
         throws CoderException, IOException {
       if (value == null) {
         outStream.write(2);
@@ -61,7 +61,7 @@ public void encode(@Nullable Boolean value, OutputStream outStream, Context cont
     @Override
     @Nullable
     public Boolean decode(
-        InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+        InputStream inStream)
         throws CoderException, IOException {
       int value = inStream.read();
       if (value == 0) {
@@ -110,7 +110,7 @@ private static class ObjectIdentityBooleanCoder extends StructuredCoder<ObjectId
 
     @Override
     public void encode(
-        @Nullable ObjectIdentityBoolean value, OutputStream outStream, Context context)
+        @Nullable ObjectIdentityBoolean value, OutputStream outStream)
         throws CoderException, IOException {
       if (value == null) {
         outStream.write(2);
@@ -124,7 +124,7 @@ public void encode(
     @Override
     @Nullable
     public ObjectIdentityBoolean decode(
-        InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+        InputStream inStream)
         throws CoderException, IOException {
       int value = inStream.read();
       if (value == 0) {
@@ -213,13 +213,13 @@ public void testGenericStandardCoder() throws Exception {
   private static class Foo<T> extends StructuredCoder<T> {
 
     @Override
-    public void encode(T value, OutputStream outStream, Coder.Context context)
+    public void encode(T value, OutputStream outStream)
         throws CoderException, IOException {
       throw new UnsupportedOperationException();
     }
 
     @Override
-    public T decode(InputStream inStream, Coder.Context context)
+    public T decode(InputStream inStream)
         throws CoderException, IOException {
       throw new UnsupportedOperationException();
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
index 83f348c2678f..37db4ef84824 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
@@ -98,12 +98,12 @@ public static NotSerializableObjectCoder of() {
     }
 
     @Override
-    public void encode(NotSerializableObject value, OutputStream outStream, Context context)
+    public void encode(NotSerializableObject value, OutputStream outStream)
         throws CoderException, IOException {
     }
 
     @Override
-    public NotSerializableObject decode(InputStream inStream, Context context)
+    public NotSerializableObject decode(InputStream inStream)
         throws CoderException, IOException {
       return new NotSerializableObject();
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
index db5ff2e3049b..375be33944cb 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
@@ -153,11 +153,11 @@ private static class NotSerializableClass {
 
   private static class NotSerializableClassCoder extends AtomicCoder<NotSerializableClass> {
     @Override
-    public void encode(NotSerializableClass value, OutputStream outStream, Coder.Context context) {
+    public void encode(NotSerializableClass value, OutputStream outStream) {
     }
 
     @Override
-    public NotSerializableClass decode(InputStream inStream, Coder.Context context) {
+    public NotSerializableClass decode(InputStream inStream) {
       return new NotSerializableClass();
     }
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java
index 546683b44e36..3939800f9a4c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java
@@ -75,14 +75,14 @@ public void unencodableWindowFails() {
   private static class FailingCoder extends AtomicCoder<BoundedWindow> {
     @Override
     public void encode(
-        BoundedWindow value, OutputStream outStream, Context context)
+        BoundedWindow value, OutputStream outStream)
         throws CoderException, IOException {
       throw new CoderException("Test Encode Exception");
     }
 
     @Override
     public BoundedWindow decode(
-        InputStream inStream, Context context) throws CoderException, IOException {
+        InputStream inStream) throws CoderException, IOException {
       throw new CoderException("Test Decode Exception");
     }
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
index 8a4d56365410..33c652a523af 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
@@ -335,12 +335,23 @@ public static UserStringCoder of() {
 
     private static final UserStringCoder INSTANCE = new UserStringCoder();
 
+    @Override
+    public void encode(UserString value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(UserString value, OutputStream outStream, Context context)
         throws CoderException, IOException {
       StringUtf8Coder.of().encode(value.strValue, outStream, context);
     }
 
+    @Override
+    public UserString decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public UserString decode(InputStream inStream, Context context)
         throws CoderException, IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
index e4b016be10bb..bd8aee405bf8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
@@ -875,6 +875,12 @@ public Coder<CountSum> getAccumulatorCoder(
      * A {@link Coder} for {@link CountSum}.
      */
     private class CountSumCoder extends AtomicCoder<CountSum> {
+      @Override
+      public void encode(CountSum value, OutputStream outStream, OutputStream outStream)
+          throws CoderException, IOException {
+        encode(outStream, outStream, Context.NESTED);
+      }
+
       @Override
       public void encode(CountSum value, OutputStream outStream,
           Context context) throws CoderException, IOException {
@@ -882,6 +888,11 @@ public void encode(CountSum value, OutputStream outStream,
         DOUBLE_CODER.encode(value.sum, outStream, context);
       }
 
+      @Override
+      public CountSum decode(InputStream inStream) throws CoderException, IOException {
+        return decode(inStream, Coder.Context.NESTED);
+      }
+
       @Override
       public CountSum decode(InputStream inStream, Coder.Context context)
           throws CoderException, IOException {
@@ -924,12 +935,23 @@ public Accumulator(String value) {
 
       public static Coder<Accumulator> getCoder() {
         return new AtomicCoder<Accumulator>() {
+          @Override
+          public void encode(Accumulator accumulator, OutputStream outStream)
+              throws CoderException, IOException {
+            encode(accumulator, outStream, Coder.Context.NESTED);
+          }
+
           @Override
           public void encode(Accumulator accumulator, OutputStream outStream, Coder.Context context)
               throws CoderException, IOException {
             StringUtf8Coder.of().encode(accumulator.value, outStream, context);
           }
 
+          @Override
+          public Accumulator decode(InputStream inStream) throws CoderException, IOException {
+            return decode(inStream, Coder.Context.NESTED);
+          }
+
           @Override
           public Accumulator decode(InputStream inStream, Coder.Context context)
               throws CoderException, IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
index 7e8a1dd7bfcc..a05d31c29d4d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
@@ -134,11 +134,11 @@ static class Record2 extends Record {
 
   private static class RecordCoder extends AtomicCoder<Record> {
     @Override
-    public void encode(Record value, OutputStream outStream, Context context)
+    public void encode(Record value, OutputStream outStream)
         throws CoderException, IOException {}
 
     @Override
-    public Record decode(InputStream inStream, Context context) throws CoderException, IOException {
+    public Record decode(InputStream inStream) throws CoderException, IOException {
       return null;
     }
   }
@@ -207,15 +207,14 @@ static class UnserializableRecordCoder extends AtomicCoder<UnserializableRecord>
       @Override
       public void encode(
           UnserializableRecord value,
-          OutputStream outStream,
-          org.apache.beam.sdk.coders.Coder.Context context)
+          OutputStream outStream)
           throws CoderException, IOException {
         stringCoder.encode(value.myString, outStream);
       }
 
       @Override
       public UnserializableRecord decode(
-          InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+          InputStream inStream)
           throws CoderException, IOException {
         return new UnserializableRecord(stringCoder.decode(inStream));
       }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
index aba33ebe8b5a..0cd885ca2557 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
@@ -469,13 +469,13 @@ public static DeterministicKeyCoder of() {
     private DeterministicKeyCoder() {}
 
     @Override
-    public void encode(BadEqualityKey value, OutputStream outStream, Context context)
+    public void encode(BadEqualityKey value, OutputStream outStream)
         throws IOException {
       new DataOutputStream(outStream).writeLong(value.key);
     }
 
     @Override
-    public BadEqualityKey decode(InputStream inStream, Context context)
+    public BadEqualityKey decode(InputStream inStream)
         throws IOException {
       return new BadEqualityKey(new DataInputStream(inStream).readLong());
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index d2cb980f3a75..3697211091a4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -986,12 +986,12 @@ public static TestDummyCoder of() {
     }
 
     @Override
-    public void encode(TestDummy value, OutputStream outStream, Context context)
+    public void encode(TestDummy value, OutputStream outStream)
         throws CoderException, IOException {
     }
 
     @Override
-    public TestDummy decode(InputStream inStream, Context context)
+    public TestDummy decode(InputStream inStream)
         throws CoderException, IOException {
       return new TestDummy();
     }
@@ -1089,12 +1089,23 @@ public static MyIntegerCoder of() {
       return INSTANCE;
     }
 
+    @Override
+    public void encode(MyInteger value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(MyInteger value, OutputStream outStream, Context context)
         throws CoderException, IOException {
       delegate.encode(value.getValue(), outStream, context);
     }
 
+    @Override
+    public MyInteger decode(InputStream inStream) throws CoderException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public MyInteger decode(InputStream inStream, Context context) throws CoderException,
         IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
index 84f3d69dd10a..cdd03d9714fd 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java
@@ -506,12 +506,23 @@ public void processElement(ProcessContext c) {
   }
 
   private static class NonDeterministicStringCoder extends AtomicCoder<String> {
+    @Override
+    public void encode(String value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Coder.Context.NESTED);
+    }
+
     @Override
     public void encode(String value, OutputStream outStream, Coder.Context context)
         throws CoderException, IOException {
       StringUtf8Coder.of().encode(value, outStream, context);
     }
 
+    @Override
+    public String decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Coder.Context.NESTED);
+    }
+
     @Override
     public String decode(InputStream inStream, Coder.Context context)
         throws CoderException, IOException {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index 489493ab22a8..a8cd35ed7066 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -277,10 +277,10 @@ public static SomeRestrictionCoder of() {
     }
 
     @Override
-    public void encode(SomeRestriction value, OutputStream outStream, Context context) {}
+    public void encode(SomeRestriction value, OutputStream outStream) {}
 
     @Override
-    public SomeRestriction decode(InputStream inStream, Context context) {
+    public SomeRestriction decode(InputStream inStream) {
       return null;
     }
   }
@@ -400,10 +400,10 @@ public static CoderForDefaultTracker of() {
 
     @Override
     public void encode(
-        RestrictionWithDefaultTracker value, OutputStream outStream, Context context) {}
+        RestrictionWithDefaultTracker value, OutputStream outStream) {}
 
     @Override
-    public RestrictionWithDefaultTracker decode(InputStream inStream, Context context) {
+    public RestrictionWithDefaultTracker decode(InputStream inStream) {
       return null;
     }
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java
index 7230a8bbaae4..f36e5e1f6cab 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CoderUtilsTest.java
@@ -50,12 +50,12 @@ public static TestCoder of() {
     }
 
     @Override
-    public void encode(Integer value, OutputStream outStream, Context context) {
+    public void encode(Integer value, OutputStream outStream) {
       throw new RuntimeException("not expecting to be called");
     }
 
     @Override
-    public Integer decode(InputStream inStream, Context context) {
+    public Integer decode(InputStream inStream) {
       throw new RuntimeException("not expecting to be called");
     }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
index 6ba1d4a8af70..9a807305a254 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java
@@ -89,12 +89,12 @@ private static class UnserializableCoderByJava extends AtomicCoder<Object> {
     private final Object unserializableField = new Object();
 
     @Override
-    public void encode(Object value, OutputStream outStream, Context context)
+    public void encode(Object value, OutputStream outStream)
         throws CoderException, IOException {
     }
 
     @Override
-    public Object decode(InputStream inStream, Context context)
+    public Object decode(InputStream inStream)
         throws CoderException, IOException {
       return unserializableField;
     }
diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
index 325c69dfee8e..73c79774e03f 100644
--- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
+++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java
@@ -48,6 +48,12 @@ public static ByteStringCoder of() {
 
   private ByteStringCoder() {}
 
+  @Override
+  public void encode(ByteString value, OutputStream outStream)
+      throws IOException, CoderException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(ByteString value, OutputStream outStream, Context context)
       throws IOException, CoderException {
@@ -62,6 +68,11 @@ public void encode(ByteString value, OutputStream outStream, Context context)
     value.writeTo(outStream);
   }
 
+  @Override
+  public ByteString decode(InputStream inStream) throws IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public ByteString decode(InputStream inStream, Context context) throws IOException {
     if (context.isWholeStream) {
diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java
index 968a2fa06230..f73bf2b09736 100644
--- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java
+++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java
@@ -167,6 +167,12 @@ public ProtoCoder<T> withExtensionsFrom(Class<?>... moreExtensionHosts) {
     return withExtensionsFrom(Arrays.asList(moreExtensionHosts));
   }
 
+  @Override
+  public void encode(T value, OutputStream outStream)
+      throws IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(T value, OutputStream outStream, Context context) throws IOException {
     if (value == null) {
@@ -179,6 +185,11 @@ public void encode(T value, OutputStream outStream, Context context) throws IOEx
     }
   }
 
+  @Override
+  public T decode(InputStream inStream) throws IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public T decode(InputStream inStream, Context context) throws IOException {
     if (context.isWholeStream) {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
index 33b9f779ab9d..f034a030da60 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestinationCoder.java
@@ -38,7 +38,7 @@ public static TableDestinationCoder of() {
   }
 
   @Override
-  public void encode(TableDestination value, OutputStream outStream, Context context)
+  public void encode(TableDestination value, OutputStream outStream)
       throws IOException {
     if (value == null) {
       throw new CoderException("cannot encode a null value");
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
index 8ae75c56581e..c4707da19b2e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
@@ -37,6 +37,12 @@ public static TableRowInfoCoder of() {
     return INSTANCE;
   }
 
+  @Override
+  public void encode(TableRowInfo value, OutputStream outStream)
+      throws IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(TableRowInfo value, OutputStream outStream, Context context)
       throws IOException {
@@ -47,6 +53,11 @@ public void encode(TableRowInfo value, OutputStream outStream, Context context)
     idCoder.encode(value.uniqueId, outStream, context);
   }
 
+  @Override
+  public TableRowInfo decode(InputStream inStream) throws IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public TableRowInfo decode(InputStream inStream, Context context)
       throws IOException {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
index cfec991d7a3a..e4b6f1fde072 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowJsonCoder.java
@@ -37,6 +37,12 @@ public static TableRowJsonCoder of() {
     return INSTANCE;
   }
 
+  @Override
+  public void encode(TableRow value, OutputStream outStream)
+      throws IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(TableRow value, OutputStream outStream, Context context)
       throws IOException {
@@ -44,6 +50,11 @@ public void encode(TableRow value, OutputStream outStream, Context context)
     StringUtf8Coder.of().encode(strValue, outStream, context);
   }
 
+  @Override
+  public TableRow decode(InputStream inStream) throws IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public TableRow decode(InputStream inStream, Context context)
       throws IOException {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
index 9e83271a5865..f0140392f4cb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java
@@ -101,7 +101,7 @@ public static <DestinationT> ResultCoder<DestinationT> of(
     }
 
     @Override
-    public void encode(Result<DestinationT> value, OutputStream outStream, Context context)
+    public void encode(Result<DestinationT> value, OutputStream outStream)
         throws IOException {
       if (value == null) {
         throw new CoderException("cannot encode a null value");
@@ -112,7 +112,7 @@ public void encode(Result<DestinationT> value, OutputStream outStream, Context c
     }
 
     @Override
-    public Result<DestinationT> decode(InputStream inStream, Context context) throws IOException {
+    public Result<DestinationT> decode(InputStream inStream) throws IOException {
       String filename = stringCoder.decode(inStream);
       long fileByteSize = longCoder.decode(inStream);
       DestinationT destination = destinationCoder.decode(inStream);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java
index d120f7225fe5..5df2bcf45734 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java
@@ -33,12 +33,23 @@ public static PubsubMessagePayloadOnlyCoder of() {
     return new PubsubMessagePayloadOnlyCoder();
   }
 
+  @Override
+  public void encode(PubsubMessage value, OutputStream outStream)
+      throws IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(PubsubMessage value, OutputStream outStream, Context context)
       throws IOException {
     PAYLOAD_CODER.encode(value.getPayload(), outStream, context);
   }
 
+  @Override
+  public PubsubMessage decode(InputStream inStream) throws IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public PubsubMessage decode(InputStream inStream, Context context) throws IOException {
     return new PubsubMessage(
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
index 5907c9e7a214..bcf7656cbee1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoder.java
@@ -45,12 +45,23 @@ public static PubsubMessageWithAttributesCoder of() {
     return new PubsubMessageWithAttributesCoder();
   }
 
+  @Override
+  public void encode(PubsubMessage value, OutputStream outStream)
+      throws IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   public void encode(PubsubMessage value, OutputStream outStream, Context context)
       throws IOException {
     PAYLOAD_CODER.encode(value.getPayload(), outStream);
     ATTRIBUTES_CODER.encode(value.getAttributeMap(), outStream, context);
   }
 
+  @Override
+  public PubsubMessage decode(InputStream inStream) throws IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public PubsubMessage decode(InputStream inStream, Context context) throws IOException {
     byte[] payload = PAYLOAD_CODER.decode(inStream);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
index ae320c73a05b..ad38e2800f9e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java
@@ -108,7 +108,7 @@ private static class OutgoingMessageCoder extends AtomicCoder<OutgoingMessage> {
 
     @Override
     public void encode(
-        OutgoingMessage value, OutputStream outStream, Context context)
+        OutgoingMessage value, OutputStream outStream)
         throws CoderException, IOException {
       ByteArrayCoder.of().encode(value.elementBytes, outStream);
       ATTRIBUTES_CODER.encode(value.attributes, outStream);
@@ -118,7 +118,7 @@ public void encode(
 
     @Override
     public OutgoingMessage decode(
-        InputStream inStream, Context context) throws CoderException, IOException {
+        InputStream inStream) throws CoderException, IOException {
       byte[] elementBytes = ByteArrayCoder.of().decode(inStream);
       Map<String, String> attributes = ATTRIBUTES_CODER.decode(inStream);
       long timestampMsSinceEpoch = BigEndianLongCoder.of().decode(inStream);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index e53976e5ffb8..db8c1b74e66d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -368,6 +368,12 @@ public static <T> PubsubCheckpointCoder<T> of() {
 
     private PubsubCheckpointCoder() {}
 
+    @Override
+    public void encode(PubsubCheckpoint value, OutputStream outStream)
+        throws IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(PubsubCheckpoint value, OutputStream outStream, Context context)
         throws IOException {
@@ -378,6 +384,11 @@ public void encode(PubsubCheckpoint value, OutputStream outStream, Context conte
       LIST_CODER.encode(value.notYetReadIds, outStream, context);
     }
 
+    @Override
+    public PubsubCheckpoint decode(InputStream inStream) throws IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public PubsubCheckpoint decode(InputStream inStream, Context context) throws IOException {
       String path = SUBSCRIPTION_PATH_CODER.decode(inStream);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index d60c7214b701..70d53779f281 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -747,12 +747,23 @@ public int hashCode() {
    * Coder for @link{PartitionedGlobalWindow}.
    */
   private static class PartitionedGlobalWindowCoder extends AtomicCoder<PartitionedGlobalWindow> {
+    @Override
+    public void encode(PartitionedGlobalWindow window, OutputStream outStream)
+        throws IOException, CoderException {
+      encode(window, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(PartitionedGlobalWindow window, OutputStream outStream, Context context)
         throws IOException, CoderException {
       StringUtf8Coder.of().encode(window.value, outStream, context);
     }
 
+    @Override
+    public PartitionedGlobalWindow decode(InputStream inStream) throws IOException, CoderException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public PartitionedGlobalWindow decode(InputStream inStream, Context context)
         throws IOException, CoderException {
diff --git a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
index 8fddfe004229..8d2598ad3712 100644
--- a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
+++ b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/WritableCoder.java
@@ -68,13 +68,13 @@ public WritableCoder(Class<T> type) {
   }
 
   @Override
-  public void encode(T value, OutputStream outStream, Context context) throws IOException {
+  public void encode(T value, OutputStream outStream) throws IOException {
     value.write(new DataOutputStream(outStream));
   }
 
   @SuppressWarnings("unchecked")
   @Override
-  public T decode(InputStream inStream, Context context) throws IOException {
+  public T decode(InputStream inStream) throws IOException {
     try {
       if (type == NullWritable.class) {
         // NullWritable has no default constructor
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
index 7cc043ce34db..501fe09259c8 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseMutationCoder.java
@@ -44,16 +44,14 @@ public static HBaseMutationCoder of() {
   }
 
   @Override
-  public void encode(Mutation mutation, OutputStream outStream,
-                     Coder.Context context) throws IOException {
+  public void encode(Mutation mutation, OutputStream outStream) throws IOException {
     MutationType type = getType(mutation);
     MutationProto proto = ProtobufUtil.toMutation(type, mutation);
     proto.writeDelimitedTo(outStream);
   }
 
   @Override
-  public Mutation decode(InputStream inStream,
-                         Coder.Context context) throws IOException {
+  public Mutation decode(InputStream inStream) throws IOException {
     return ProtobufUtil.toMutation(MutationProto.parseDelimitedFrom(inStream));
   }
 
diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
index 24a5f7f090aa..1d066351d1aa 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseResultCoder.java
@@ -41,13 +41,13 @@ public static HBaseResultCoder of() {
   }
 
   @Override
-  public void encode(Result value, OutputStream outputStream, Coder.Context context)
+  public void encode(Result value, OutputStream outputStream)
           throws IOException {
     ProtobufUtil.toResult(value).writeDelimitedTo(outputStream);
   }
 
   @Override
-  public Result decode(InputStream inputStream, Coder.Context context)
+  public Result decode(InputStream inputStream)
       throws IOException {
     return ProtobufUtil.toResult(ClientProtos.Result.parseDelimitedFrom(inputStream));
   }
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index ba84c2a3484a..e21945fd7f6d 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -1597,13 +1597,13 @@ public void populateDisplayData(DisplayData.Builder builder) {
 
   private static class NullOnlyCoder<T> extends AtomicCoder<T> {
     @Override
-    public void encode(T value, OutputStream outStream, Context context) {
+    public void encode(T value, OutputStream outStream) {
       checkArgument(value == null, "Can only encode nulls");
       // Encode as no bytes.
     }
 
     @Override
-    public T decode(InputStream inStream, Context context) {
+    public T decode(InputStream inStream) {
       return null;
     }
   }
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
index d838a0d7943f..197106011e1f 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecordCoder.java
@@ -49,6 +49,12 @@ public KafkaRecordCoder(Coder<K> keyCoder, Coder<V> valueCoder) {
     this.kvCoder = KvCoder.of(keyCoder, valueCoder);
   }
 
+  @Override
+  public void encode(KafkaRecord<K, V> value, OutputStream outStream)
+      throws CoderException, IOException {
+    encode(value, outStream, Context.NESTED);
+  }
+
   @Override
   public void encode(KafkaRecord<K, V> value, OutputStream outStream, Context context)
                          throws CoderException, IOException {
@@ -60,6 +66,11 @@ public void encode(KafkaRecord<K, V> value, OutputStream outStream, Context cont
     kvCoder.encode(value.getKV(), outStream, context);
   }
 
+  @Override
+  public KafkaRecord<K, V> decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public KafkaRecord<K, V> decode(InputStream inStream, Context context)
                                       throws CoderException, IOException {
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
index c6a017461ad4..f233e27d064f 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecordCoder.java
@@ -43,7 +43,7 @@ public static KinesisRecordCoder of() {
     }
 
     @Override
-    public void encode(KinesisRecord value, OutputStream outStream, Context context) throws
+    public void encode(KinesisRecord value, OutputStream outStream) throws
             IOException {
         BYTE_ARRAY_CODER.encode(value.getData().array(), outStream);
         STRING_CODER.encode(value.getSequenceNumber(), outStream);
@@ -56,7 +56,7 @@ public void encode(KinesisRecord value, OutputStream outStream, Context context)
     }
 
     @Override
-    public KinesisRecord decode(InputStream inStream, Context context) throws IOException {
+    public KinesisRecord decode(InputStream inStream) throws IOException {
         ByteBuffer data = ByteBuffer.wrap(BYTE_ARRAY_CODER.decode(inStream));
         String sequenceNumber = STRING_CODER.decode(inStream);
         String partitionKey = STRING_CODER.decode(inStream);
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
index 5b2ec0251484..d4c0440cc478 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java
@@ -88,15 +88,8 @@ public static <T> JAXBCoder<T> of(Class<T> jaxbClass) {
   }
 
   @Override
-  public void encode(T value, OutputStream outStream) throws IOException {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    try {
-      jaxbMarshaller.get().marshal(value, baos);
-    } catch (JAXBException e) {
-      throw new CoderException(e);
-    }
-    VarInt.encode(baos.size(), outStream);
-    baos.writeTo(outStream);
+  public void encode(T value, OutputStream outStream) throws CoderException, IOException {
+    encode(value, outStream, Context.NESTED);
   }
 
   @Override
@@ -109,10 +102,22 @@ public void encode(T value, OutputStream outStream, Context context)
         throw new CoderException(e);
       }
     } else {
-      encode(value, outStream);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      try {
+        jaxbMarshaller.get().marshal(value, baos);
+      } catch (JAXBException e) {
+        throw new CoderException(e);
+      }
+      VarInt.encode(baos.size(), outStream);
+      baos.writeTo(outStream);
     }
   }
 
+  @Override
+  public T decode(InputStream inStream) throws CoderException, IOException {
+    return decode(inStream, Context.NESTED);
+  }
+
   @Override
   public T decode(InputStream inStream, Context context) throws CoderException, IOException {
     try {
diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
index 5386a613fcbc..c175e4a3444a 100644
--- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
+++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java
@@ -177,19 +177,28 @@ public TestCoder(JAXBCoder<TestType> jaxbCoder) {
       this.jaxbCoder = jaxbCoder;
     }
 
+    @Override
+    public void encode(TestType value, OutputStream outStream)
+        throws CoderException, IOException {
+      encode(value, outStream, Context.NESTED);
+    }
+
     @Override
     public void encode(TestType value, OutputStream outStream, Context context)
         throws CoderException, IOException {
-      Context nestedContext = context.nested();
       VarIntCoder.of().encode(3, outStream);
       jaxbCoder.encode(value, outStream);
       VarLongCoder.of().encode(22L, outStream, context);
     }
 
+    @Override
+    public TestType decode(InputStream inStream) throws CoderException, IOException {
+      return decode(inStream, Context.NESTED);
+    }
+
     @Override
     public TestType decode(InputStream inStream, Context context)
         throws CoderException, IOException {
-      Context nestedContext = context.nested();
       VarIntCoder.of().decode(inStream);
       TestType result = jaxbCoder.decode(inStream);
       VarLongCoder.of().decode(inStream, context);

From 45e09b2df7a97cfbbe0d8013c16cfdd7a55afbde Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Fri, 5 May 2017 17:27:13 -0700
Subject: [PATCH 128/387] get it compiling

---
 .../runners/dataflow/BatchViewOverrides.java  |  4 +--
 .../apache/beam/sdk/transforms/Combine.java   |  3 +-
 .../beam/sdk/transforms/join/CoGbkResult.java |  2 +-
 .../sdk/transforms/windowing/PaneInfo.java    |  1 -
 .../apache/beam/sdk/util/WindowedValue.java   |  3 +-
 .../beam/sdk/coders/CoderRegistryTest.java    |  6 ++++
 .../beam/sdk/testing/CoderPropertiesTest.java | 36 +++++++++----------
 .../sdk/testing/SerializableMatchersTest.java |  1 -
 .../beam/sdk/transforms/CombineTest.java      | 19 +++-------
 .../apache/beam/sdk/transforms/ParDoTest.java | 17 ++-------
 10 files changed, 37 insertions(+), 55 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index 34609dfecce3..d640f6effd00 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -1351,9 +1351,9 @@ public static <K, V1, V2> TransformedMapCoder<K, V1, V2> of(
     }
 
     @Override
-    public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream, OutputStream outStream)
+    public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream)
         throws CoderException, IOException {
-      encode(outStream, outStream, Coder.Context.NESTED);
+      encode(value, outStream, Coder.Context.NESTED);
     }
 
     @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 7e435647ed15..9e1cc7113b80 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -2001,7 +2001,8 @@ public void encode(
         }
 
         @Override
-        public InputOrAccum<InputT, AccumT> decode(InputStream inStream) throws CoderException, IOException {
+        public InputOrAccum<InputT, AccumT> decode(InputStream inStream)
+            throws CoderException, IOException {
           return decode(inStream, Coder.Context.NESTED);
         }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
index d42de829c789..877bb077c78c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java
@@ -264,7 +264,7 @@ public CoGbkResult decode(
       }
       List<Iterable<?>> valueMap = Lists.newArrayListWithExpectedSize(schema.size());
       for (int unionTag = 0; unionTag < schema.size(); unionTag++) {
-        valueMap.add(tagListCoder(unionTag).decode(inStream, Coder.Context.NESTED));
+        valueMap.add(tagListCoder(unionTag).decode(inStream));
       }
       return new CoGbkResult(schema, valueMap);
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
index 75df220ab873..1e9a187f6b6c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
@@ -27,7 +27,6 @@
 import java.io.OutputStream;
 import java.util.Objects;
 import org.apache.beam.sdk.coders.AtomicCoder;
-import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
index 963886bee9d6..444521adc8f7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
@@ -646,8 +646,7 @@ public void encode(WindowedValue<T> windowedElem,
                        OutputStream outStream,
                        Context context)
         throws CoderException, IOException {
-      InstantCoder.of().encode(
-          windowedElem.getTimestamp(), outStream, nestedContext);
+      InstantCoder.of().encode(windowedElem.getTimestamp(), outStream);
       windowsCoder.encode(windowedElem.getWindows(), outStream);
       PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream);
       valueCoder.encode(windowedElem.getValue(), outStream, context);
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
index c883ca021366..b199a06caa59 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
@@ -438,6 +438,12 @@ private static class AutoRegistrationClass {
 
   private static class AutoRegistrationClassCoder extends CustomCoder<AutoRegistrationClass> {
     private static final AutoRegistrationClassCoder INSTANCE = new AutoRegistrationClassCoder();
+
+    public void encode(AutoRegistrationClass value, OutputStream outStream) {}
+
+    public AutoRegistrationClass decode(InputStream inStream) {
+      return null;
+    }
   }
 
   @Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java
index 164d22131446..ce78411880b6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java
@@ -50,15 +50,15 @@ public void testGoodCoderIsDeterministic() throws Exception {
   /** A coder that says it is not deterministic but actually is. */
   public static class NonDeterministicCoder extends AtomicCoder<String> {
     @Override
-    public void encode(String value, OutputStream outStream, Context context)
+    public void encode(String value, OutputStream outStream)
         throws CoderException, IOException {
-      StringUtf8Coder.of().encode(value, outStream, context);
+      StringUtf8Coder.of().encode(value, outStream);
     }
 
     @Override
-    public String decode(InputStream inStream, Context context)
+    public String decode(InputStream inStream)
         throws CoderException, IOException {
-      return StringUtf8Coder.of().decode(inStream, context);
+      return StringUtf8Coder.of().decode(inStream);
     }
 
     public void verifyDeterministic() throws NonDeterministicException {
@@ -96,15 +96,15 @@ public BadDeterminsticCoder() {
     }
 
     @Override
-    public void encode(String value, OutputStream outStream, Context context)
+    public void encode(String value, OutputStream outStream)
         throws IOException, CoderException {
-      StringUtf8Coder.of().encode(value + System.nanoTime(), outStream, context);
+      StringUtf8Coder.of().encode(value + System.nanoTime(), outStream);
     }
 
     @Override
-    public String decode(InputStream inStream, Context context)
+    public String decode(InputStream inStream)
         throws CoderException, IOException {
-      return StringUtf8Coder.of().decode(inStream, context);
+      return StringUtf8Coder.of().decode(inStream);
     }
 
     @Override
@@ -136,16 +136,16 @@ public StateChangingSerializingCoder() {
     }
 
     @Override
-    public void encode(String value, OutputStream outStream, Context context)
+    public void encode(String value, OutputStream outStream)
         throws CoderException, IOException {
       changedState += 1;
-      StringUtf8Coder.of().encode(value + Strings.repeat("A", changedState), outStream, context);
+      StringUtf8Coder.of().encode(value + Strings.repeat("A", changedState), outStream);
     }
 
     @Override
-    public String decode(InputStream inStream, Context context)
+    public String decode(InputStream inStream)
         throws CoderException, IOException {
-      String decodedValue = StringUtf8Coder.of().decode(inStream, context);
+      String decodedValue = StringUtf8Coder.of().decode(inStream);
       return decodedValue.substring(0, decodedValue.length() - changedState);
     }
 
@@ -180,18 +180,18 @@ public ForgetfulSerializingCoder(int lostState) {
     }
 
     @Override
-    public void encode(String value, OutputStream outStream, Context context)
+    public void encode(String value, OutputStream outStream)
         throws CoderException, IOException {
       if (lostState == 0) {
         throw new RuntimeException("I forgot something...");
       }
-      StringUtf8Coder.of().encode(value, outStream, context);
+      StringUtf8Coder.of().encode(value, outStream);
     }
 
     @Override
-    public String decode(InputStream inStream, Context context)
+    public String decode(InputStream inStream)
         throws CoderException, IOException {
-      return StringUtf8Coder.of().decode(inStream, context);
+      return StringUtf8Coder.of().decode(inStream);
     }
 
     @Override
@@ -216,12 +216,12 @@ public void testBadCoderThatDependsOnStateThatIsLost() throws Exception {
   /** A coder which closes the underlying stream during encoding and decoding. */
   public static class ClosingCoder extends AtomicCoder<String> {
     @Override
-    public void encode(String value, OutputStream outStream, Context context) throws IOException {
+    public void encode(String value, OutputStream outStream) throws IOException {
       outStream.close();
     }
 
     @Override
-    public String decode(InputStream inStream, Context context) throws IOException {
+    public String decode(InputStream inStream) throws IOException {
       inStream.close();
       return null;
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
index 375be33944cb..6b176965620a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java
@@ -30,7 +30,6 @@
 import java.io.OutputStream;
 import java.io.Serializable;
 import org.apache.beam.sdk.coders.AtomicCoder;
-import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.values.KV;
 import org.hamcrest.Matchers;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
index bd8aee405bf8..dc9788f69a1e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
@@ -876,28 +876,17 @@ public Coder<CountSum> getAccumulatorCoder(
      */
     private class CountSumCoder extends AtomicCoder<CountSum> {
       @Override
-      public void encode(CountSum value, OutputStream outStream, OutputStream outStream)
+      public void encode(CountSum value, OutputStream outStream)
           throws CoderException, IOException {
-        encode(outStream, outStream, Context.NESTED);
-      }
-
-      @Override
-      public void encode(CountSum value, OutputStream outStream,
-          Context context) throws CoderException, IOException {
         LONG_CODER.encode(value.count, outStream);
-        DOUBLE_CODER.encode(value.sum, outStream, context);
-      }
-
-      @Override
-      public CountSum decode(InputStream inStream) throws CoderException, IOException {
-        return decode(inStream, Coder.Context.NESTED);
+        DOUBLE_CODER.encode(value.sum, outStream);
       }
 
       @Override
-      public CountSum decode(InputStream inStream, Coder.Context context)
+      public CountSum decode(InputStream inStream)
           throws CoderException, IOException {
         long count = LONG_CODER.decode(inStream);
-        double sum = DOUBLE_CODER.decode(inStream, context);
+        double sum = DOUBLE_CODER.decode(inStream);
         return new CountSum(count, sum);
       }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 3697211091a4..ef27f4c03446 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -1092,24 +1092,13 @@ public static MyIntegerCoder of() {
     @Override
     public void encode(MyInteger value, OutputStream outStream)
         throws CoderException, IOException {
-      encode(value, outStream, Context.NESTED);
+      delegate.encode(value.getValue(), outStream);
     }
 
     @Override
-    public void encode(MyInteger value, OutputStream outStream, Context context)
-        throws CoderException, IOException {
-      delegate.encode(value.getValue(), outStream, context);
-    }
-
-    @Override
-    public MyInteger decode(InputStream inStream) throws CoderException {
-      return decode(inStream, Context.NESTED);
-    }
-
-    @Override
-    public MyInteger decode(InputStream inStream, Context context) throws CoderException,
+    public MyInteger decode(InputStream inStream) throws CoderException,
         IOException {
-      return new MyInteger(delegate.decode(inStream, context));
+      return new MyInteger(delegate.decode(inStream));
     }
   }
 

From 44867300cb36dcad1cdfba70dbe093ec50a14388 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Fri, 5 May 2017 17:35:35 -0700
Subject: [PATCH 129/387] Remove en/decodeOuter and default encode/decode
 methods.

Now only the context-free encode() and decode() methods are abstract.
---
 .../org/apache/beam/sdk/coders/Coder.java     | 35 ++++----------
 .../apache/beam/sdk/coders/CustomCoder.java   | 47 -------------------
 .../beam/sdk/coders/StructuredCoder.java      | 47 -------------------
 3 files changed, 8 insertions(+), 121 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index d140e896e0a0..ec8a72d02c89 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -125,18 +125,6 @@ public String toString() {
   public abstract void encode(T value, OutputStream outStream)
       throws CoderException, IOException;
 
-  /**
-   * Encodes the given value of type {@code T} onto the given output stream
-   * in the outer context.
-   *
-   * @throws IOException if writing to the {@code OutputStream} fails
-   * for some reason
-   * @throws CoderException if the value could not be encoded for some reason
-   */
-  @Deprecated
-  public abstract void encodeOuter(T value, OutputStream outStream)
-      throws CoderException, IOException;
-
   /**
    * Encodes the given value of type {@code T} onto the given output stream
    * in the given context.
@@ -146,8 +134,10 @@ public abstract void encodeOuter(T value, OutputStream outStream)
    * @throws CoderException if the value could not be encoded for some reason
    */
   @Deprecated
-  public abstract void encode(T value, OutputStream outStream, Context context)
-      throws CoderException, IOException;
+  public void encode(T value, OutputStream outStream, Context context)
+      throws CoderException, IOException {
+    encode(value, outStream);
+  }
 
   /**
    * Decodes a value of type {@code T} from the given input stream in
@@ -159,17 +149,6 @@ public abstract void encode(T value, OutputStream outStream, Context context)
    */
   public abstract T decode(InputStream inStream) throws CoderException, IOException;
 
-  /**
-   * Decodes a value of type {@code T} from the given input stream in
-   * the outer context.  Returns the decoded value.
-   *
-   * @throws IOException if reading from the {@code InputStream} fails
-   * for some reason
-   * @throws CoderException if the value could not be decoded for some reason
-   */
-  @Deprecated
-  public abstract T decodeOuter(InputStream inStream) throws CoderException, IOException;
-
   /**
    * Decodes a value of type {@code T} from the given input stream in
    * the given context.  Returns the decoded value.
@@ -179,8 +158,10 @@ public abstract void encode(T value, OutputStream outStream, Context context)
    * @throws CoderException if the value could not be decoded for some reason
    */
   @Deprecated
-  public abstract T decode(InputStream inStream, Context context)
-      throws CoderException, IOException;
+  public T decode(InputStream inStream, Context context)
+      throws CoderException, IOException {
+    return decode(inStream);
+  }
 
   /**
    * If this is a {@code Coder} for a parameterized type, returns the
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
index edbaa7f92fc8..c58192373432 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
@@ -17,9 +17,6 @@
  */
 package org.apache.beam.sdk.coders;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
@@ -39,50 +36,6 @@
 public abstract class CustomCoder<T> extends Coder<T>
     implements Serializable {
 
-  @Override
-  public void encode(T value, OutputStream outStream)
-      throws CoderException, IOException {
-    encode(value, outStream, Coder.Context.NESTED);
-  }
-
-  @Deprecated
-  @Override
-  public void encodeOuter(T value, OutputStream outStream)
-      throws CoderException, IOException {
-    encode(value, outStream, Coder.Context.OUTER);
-  }
-
-  @Deprecated
-  public void encode(T value, OutputStream outStream, Coder.Context context)
-      throws CoderException, IOException {
-    if (context == Coder.Context.NESTED) {
-      encode(value, outStream);
-    } else {
-      encodeOuter(value, outStream);
-    }
-  }
-
-  @Override
-  public T decode(InputStream inStream) throws CoderException, IOException {
-    return decode(inStream, Coder.Context.NESTED);
-  }
-
-  @Deprecated
-  @Override
-  public T decodeOuter(InputStream inStream) throws CoderException, IOException {
-    return decode(inStream, Coder.Context.OUTER);
-  }
-
-  @Deprecated
-  public T decode(InputStream inStream, Coder.Context context)
-      throws CoderException, IOException {
-    if (context == Coder.Context.NESTED) {
-      return decode(inStream);
-    } else {
-      return decodeOuter(inStream);
-    }
-  }
-
   /**
    * {@inheritDoc}.
    *
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
index 437f10d0b1c0..42c0598aa985 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
@@ -18,9 +18,6 @@
 package org.apache.beam.sdk.coders;
 
 import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -101,50 +98,6 @@ public String toString() {
     return builder.toString();
   }
 
-  @Override
-  public void encode(T value, OutputStream outStream)
-      throws CoderException, IOException {
-    encode(value, outStream, Coder.Context.NESTED);
-  }
-
-  @Deprecated
-  @Override
-  public void encodeOuter(T value, OutputStream outStream)
-      throws CoderException, IOException {
-    encode(value, outStream, Coder.Context.OUTER);
-  }
-
-  @Deprecated
-  public void encode(T value, OutputStream outStream, Coder.Context context)
-      throws CoderException, IOException {
-    if (context == Coder.Context.NESTED) {
-      encode(value, outStream);
-    } else {
-      encodeOuter(value, outStream);
-    }
-  }
-
-  @Override
-  public T decode(InputStream inStream) throws CoderException, IOException {
-    return decode(inStream, Coder.Context.NESTED);
-  }
-
-  @Deprecated
-  @Override
-  public T decodeOuter(InputStream inStream) throws CoderException, IOException {
-    return decode(inStream, Coder.Context.OUTER);
-  }
-
-  @Deprecated
-  public T decode(InputStream inStream, Coder.Context context)
-      throws CoderException, IOException {
-    if (context == Coder.Context.NESTED) {
-      return decode(inStream);
-    } else {
-      return decodeOuter(inStream);
-    }
-  }
-
   protected void verifyDeterministic(String message, Iterable<Coder<?>> coders)
       throws NonDeterministicException {
     for (Coder<?> coder : coders) {

From 2d379ddd5ef9b0b7a54ddc70203f37cd2763f387 Mon Sep 17 00:00:00 2001
From: Lukasz Cwik <lcwik@google.com>
Date: Sun, 7 May 2017 19:41:07 -0700
Subject: [PATCH 130/387] fixup! Swap to use encode/decode in anonymous inner
 class coder and @AutoValue coder

---
 .../core/construction/PCollectionsTest.java     | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
index 2c45cbdb94b3..a114cf5b0612 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
@@ -29,10 +29,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.AtomicCoder;
-import org.apache.beam.sdk.coders.BigEndianLongCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.*;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -130,13 +127,13 @@ public void testEncodeDecodeCycle() throws Exception {
   @AutoValue
   abstract static class CustomIntCoder extends CustomCoder<Integer> {
     @Override
-    public void encode(Integer value, OutputStream outStream, Context context) throws IOException {
-      VarInt.encode(value, outStream);
+    public Integer decode(InputStream inStream) throws IOException {
+      return VarInt.decodeInt(inStream);
     }
 
     @Override
-    public Integer decode(InputStream inStream, Context context) throws IOException {
-      return VarInt.decodeInt(inStream);
+    public void encode(Integer value, OutputStream outStream) throws IOException {
+      VarInt.encode(value, outStream);
     }
   }
 
@@ -163,13 +160,13 @@ public Coder<BoundedWindow> windowCoder() {
         @Override public void verifyDeterministic() {}
 
         @Override
-        public void encode(BoundedWindow value, OutputStream outStream, Context context)
+        public void encode(BoundedWindow value, OutputStream outStream)
             throws IOException {
           VarInt.encode(value.maxTimestamp().getMillis(), outStream);
         }
 
         @Override
-        public BoundedWindow decode(InputStream inStream, Context context) throws IOException {
+        public BoundedWindow decode(InputStream inStream) throws IOException {
           final Instant ts = new Instant(VarInt.decodeLong(inStream));
           return new BoundedWindow() {
             @Override

From 4c002724c1adffc6acb1a5b424f864d451e1061c Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 10:48:40 -0700
Subject: [PATCH 131/387] Reviewer comments + a couple of extra fixes. All
 compiles.

---
 .../beam/runners/direct/DirectRunnerTest.java  |  4 ++--
 .../runners/dataflow/BatchViewOverrides.java   | 16 ++--------------
 .../runners/dataflow/internal/IsmFormat.java   | 16 ++--------------
 .../dataflow/util/CloudObjectsTest.java        |  8 ++++----
 .../org/apache/beam/sdk/io/FileBasedSink.java  | 18 +++---------------
 .../beam/sdk/coders/CoderRegistryTest.java     |  2 ++
 .../io/gcp/pubsub/PubsubUnboundedSource.java   | 18 +++---------------
 7 files changed, 18 insertions(+), 64 deletions(-)

diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index 85e55eb3a30c..943d27c07ad1 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -526,11 +526,11 @@ public void testUnencodableOutputFromUnboundedRead() {
   private static class LongNoDecodeCoder extends AtomicCoder<Long> {
     @Override
     public void encode(
-        Long value, OutputStream outStream, Context context) throws IOException {
+        Long value, OutputStream outStream) throws IOException {
     }
 
     @Override
-    public Long decode(InputStream inStream, Context context) throws IOException {
+    public Long decode(InputStream inStream) throws IOException {
       throw new CoderException("Cannot decode a long");
     }
   }
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index d640f6effd00..32a04c096095 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -1353,27 +1353,15 @@ public static <K, V1, V2> TransformedMapCoder<K, V1, V2> of(
     @Override
     public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream)
         throws CoderException, IOException {
-      encode(value, outStream, Coder.Context.NESTED);
-    }
-
-    @Override
-    public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream,
-        Coder.Context context) throws CoderException, IOException {
       transformCoder.encode(value.transform, outStream);
-      originalMapCoder.encode(value.originalMap, outStream, context);
+      originalMapCoder.encode(value.originalMap, outStream);
     }
 
     @Override
     public TransformedMap<K, V1, V2> decode(InputStream inStream) throws CoderException, IOException {
-      return decode(inStream, Coder.Context.NESTED);
-    }
-
-    @Override
-    public TransformedMap<K, V1, V2> decode(
-        InputStream inStream, Coder.Context context) throws CoderException, IOException {
       return new TransformedMap<>(
           transformCoder.decode(inStream),
-          originalMapCoder.decode(inStream, context));
+          originalMapCoder.decode(inStream));
     }
 
     @Override
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
index 8cfae816d217..0796d0805b23 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
@@ -622,32 +622,20 @@ private IsmShardCoder() {}
     @Override
     public void encode(IsmShard value, OutputStream outStream)
         throws CoderException, IOException {
-      encode(value, outStream, Coder.Context.NESTED);
-    }
-
-    @Override
-    public void encode(IsmShard value, OutputStream outStream, Coder.Context context)
-        throws CoderException, IOException {
       checkState(value.getIndexOffset() >= 0,
           "%s attempting to be written without index offset.",
           value);
       VarIntCoder.of().encode(value.getId(), outStream);
       VarLongCoder.of().encode(value.getBlockOffset(), outStream);
-      VarLongCoder.of().encode(value.getIndexOffset(), outStream, context);
+      VarLongCoder.of().encode(value.getIndexOffset(), outStream);
     }
 
     @Override
     public IsmShard decode(InputStream inStream) throws CoderException, IOException {
-      return decode(inStream, Coder.Context.NESTED);
-    }
-
-    @Override
-    public IsmShard decode(
-        InputStream inStream, Coder.Context context) throws CoderException, IOException {
       return IsmShard.of(
           VarIntCoder.of().decode(inStream),
           VarLongCoder.of().decode(inStream),
-          VarLongCoder.of().decode(inStream, context));
+          VarLongCoder.of().decode(inStream));
     }
 
     @Override
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java
index 64c0dbd8170f..59a54316d8d5 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java
@@ -171,12 +171,12 @@ private static class Record implements Serializable {}
 
   private static class ObjectCoder extends CustomCoder<Object> {
     @Override
-    public void encode(Object value, OutputStream outStream, Context context)
+    public void encode(Object value, OutputStream outStream)
         throws CoderException, IOException {
     }
 
     @Override
-    public Object decode(InputStream inStream, Context context)
+    public Object decode(InputStream inStream)
         throws CoderException, IOException {
       return new Object();
     }
@@ -197,11 +197,11 @@ public int hashCode() {
    */
   private static class ArbitraryCoder extends StructuredCoder<Record> {
     @Override
-    public void encode(Record value, OutputStream outStream, Context context)
+    public void encode(Record value, OutputStream outStream)
         throws CoderException, IOException {}
 
     @Override
-    public Record decode(InputStream inStream, Context context) throws CoderException, IOException {
+    public Record decode(InputStream inStream) throws CoderException, IOException {
       return new Record();
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 3620c22737ec..32aa9c388d89 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -949,34 +949,22 @@ public static FileResultCoder of() {
     @Override
     public void encode(FileResult value, OutputStream outStream)
         throws IOException {
-      encode(value, outStream, Context.NESTED);
-    }
-
-    @Override
-    public void encode(FileResult value, OutputStream outStream, Context context)
-        throws IOException {
       if (value == null) {
         throw new CoderException("cannot encode a null value");
       }
       stringCoder.encode(value.getFilename().toString(), outStream);
       if (value.getDestinationFilename() == null) {
-        stringCoder.encode(null, outStream, context);
+        stringCoder.encode(null, outStream);
       } else {
-        stringCoder.encode(value.getDestinationFilename().toString(), outStream, context);
+        stringCoder.encode(value.getDestinationFilename().toString(), outStream);
       }
     }
 
     @Override
     public FileResult decode(InputStream inStream) throws IOException {
-      return decode(inStream, Context.NESTED);
-    }
-
-    @Override
-    public FileResult decode(InputStream inStream, Context context)
-        throws IOException {
       String filename = stringCoder.decode(inStream);
       assert filename != null;  // fixes a compiler warning
-      @Nullable String destinationFilename = stringCoder.decode(inStream, context);
+      @Nullable String destinationFilename = stringCoder.decode(inStream);
       return new FileResult(
           FileSystems.matchNewResource(filename, false /* isDirectory */),
           destinationFilename == null
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
index b199a06caa59..d1113f7aa938 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java
@@ -439,8 +439,10 @@ private static class AutoRegistrationClass {
   private static class AutoRegistrationClassCoder extends CustomCoder<AutoRegistrationClass> {
     private static final AutoRegistrationClassCoder INSTANCE = new AutoRegistrationClassCoder();
 
+    @Override
     public void encode(AutoRegistrationClass value, OutputStream outStream) {}
 
+    @Override
     public AutoRegistrationClass decode(InputStream inStream) {
       return null;
     }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index db8c1b74e66d..e8fe7011c121 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -371,28 +371,16 @@ private PubsubCheckpointCoder() {}
     @Override
     public void encode(PubsubCheckpoint value, OutputStream outStream)
         throws IOException {
-      encode(value, outStream, Context.NESTED);
-    }
-
-    @Override
-    public void encode(PubsubCheckpoint value, OutputStream outStream, Context context)
-        throws IOException {
       SUBSCRIPTION_PATH_CODER.encode(
           value.subscriptionPath,
-          outStream,
-          context.nested());
-      LIST_CODER.encode(value.notYetReadIds, outStream, context);
+          outStream);
+      LIST_CODER.encode(value.notYetReadIds, outStream);
     }
 
     @Override
     public PubsubCheckpoint decode(InputStream inStream) throws IOException {
-      return decode(inStream, Context.NESTED);
-    }
-
-    @Override
-    public PubsubCheckpoint decode(InputStream inStream, Context context) throws IOException {
       String path = SUBSCRIPTION_PATH_CODER.decode(inStream);
-      List<String> notYetReadIds = LIST_CODER.decode(inStream, context);
+      List<String> notYetReadIds = LIST_CODER.decode(inStream);
       return new PubsubCheckpoint(path, null, null, notYetReadIds);
     }
   }

From ebd86e18a6bade77f72fca944d29b6c0308fd862 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 11:27:40 -0700
Subject: [PATCH 132/387] lint error

---
 .../beam/runners/core/construction/PCollectionsTest.java     | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
index a114cf5b0612..c38dbc052468 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
@@ -29,7 +29,10 @@
 import java.util.Collection;
 import java.util.Collections;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.*;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.testing.TestPipeline;

From d9d02f21875a0a6497d4fbfc4301d035f86ff428 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 11:54:48 -0700
Subject: [PATCH 133/387] checkstyle

---
 .../wrappers/streaming/SingletonKeyedWorkItemCoder.java        | 3 ++-
 .../wrappers/streaming/state/FlinkKeyGroupStateInternals.java  | 1 -
 .../org/apache/beam/runners/dataflow/BatchViewOverrides.java   | 3 ++-
 .../org/apache/beam/runners/dataflow/internal/IsmFormat.java   | 3 ++-
 .../fn/harness/data/BeamFnDataBufferingOutboundObserver.java   | 1 -
 .../apache/beam/fn/harness/data/BeamFnDataInboundObserver.java | 1 -
 .../harness/data/BeamFnDataBufferingOutboundObserverTest.java  | 1 -
 .../beam/fn/harness/data/BeamFnDataInboundObserverTest.java    | 1 -
 8 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
index b62fc16fbe6c..2ed2055b489f 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
@@ -81,7 +81,8 @@ public void encode(SingletonKeyedWorkItem<K, ElemT> value,
   }
 
   @Override
-  public SingletonKeyedWorkItem<K, ElemT> decode(InputStream inStream) throws CoderException, IOException {
+  public SingletonKeyedWorkItem<K, ElemT> decode(InputStream inStream)
+      throws CoderException, IOException {
     return decode(inStream, Context.NESTED);
   }
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
index 8d437d56d573..512e4efc85a0 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkKeyGroupStateInternals.java
@@ -31,7 +31,6 @@
 import org.apache.beam.runners.core.StateNamespace;
 import org.apache.beam.runners.core.StateTag;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index 32a04c096095..b4a6e6470a4c 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -1358,7 +1358,8 @@ public void encode(TransformedMap<K, V1, V2> value, OutputStream outStream)
     }
 
     @Override
-    public TransformedMap<K, V1, V2> decode(InputStream inStream) throws CoderException, IOException {
+    public TransformedMap<K, V1, V2> decode(InputStream inStream)
+        throws CoderException, IOException {
       return new TransformedMap<>(
           transformCoder.decode(inStream),
           originalMapCoder.decode(inStream));
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
index 0796d0805b23..81ac2a29a80f 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
@@ -231,7 +231,8 @@ public Coder<V> getValueCoder() {
     }
 
     @Override
-    public void encode(IsmRecord<V> value, OutputStream outStream) throws CoderException, IOException {
+    public void encode(IsmRecord<V> value, OutputStream outStream)
+        throws CoderException, IOException {
       if (value.getKeyComponents().size() != keyComponentCoders.size()) {
         throw new CoderException(String.format(
             "Expected %s key component(s) but received key component(s) %s.",
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
index 37745be9cb9f..7223e876c896 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserver.java
@@ -27,7 +27,6 @@
 import org.apache.beam.fn.v1.BeamFnApi;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
index ece87d26c8e5..ac603bde8db9 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserver.java
@@ -23,7 +23,6 @@
 import org.apache.beam.fn.harness.fn.ThrowingConsumer;
 import org.apache.beam.fn.v1.BeamFnApi;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
index 3f6ece71063b..c2b45424466a 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataBufferingOutboundObserverTest.java
@@ -33,7 +33,6 @@
 import org.apache.beam.fn.v1.BeamFnApi;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.LengthPrefixCoder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
index 4b0bf0ce932b..54aba8be12b2 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/BeamFnDataInboundObserverTest.java
@@ -34,7 +34,6 @@
 import java.util.concurrent.ExecutionException;
 import org.apache.beam.fn.v1.BeamFnApi;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.WindowedValue;

From 43037a33b1ac95326c02c97866dfce63b38308ea Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 14:56:25 -0700
Subject: [PATCH 134/387] Use latest dataflow worker.

---
 runners/google-cloud-dataflow-java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index adbd4d70bd25..9643b69878cd 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170508</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170508-pr2933</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>

From fda3a43be3277d0dca888cfa30693599d11cd5af Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 15:04:36 -0700
Subject: [PATCH 135/387] Explicitly mark Coder context as experimental as well
 as deprecated.

---
 .../java/org/apache/beam/sdk/annotations/Experimental.java     | 3 +++
 .../core/src/main/java/org/apache/beam/sdk/coders/Coder.java   | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
index 7255a01bd183..2e3a711cddfa 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
@@ -84,6 +84,9 @@ enum Kind {
     /** Metrics-related experimental APIs. */
     METRICS,
 
+    /** Experimental feature related to alternative, unnested encodings for coders. */
+    CODER_CONTEXT,
+
     /** Experimental runner APIs. Should not be used by pipeline authors. */
     CORE_RUNNERS_ONLY,
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index ec8a72d02c89..2ee532dfaa6e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -64,6 +64,7 @@
 public abstract class Coder<T> implements Serializable {
   /** The context in which encoding or decoding is being done. */
   @Deprecated
+  @Experimental(Kind.CODER_CONTEXT)
   public static class Context {
     /**
      * The outer context: the value being encoded or decoded takes
@@ -134,6 +135,7 @@ public abstract void encode(T value, OutputStream outStream)
    * @throws CoderException if the value could not be encoded for some reason
    */
   @Deprecated
+  @Experimental(Kind.CODER_CONTEXT)
   public void encode(T value, OutputStream outStream, Context context)
       throws CoderException, IOException {
     encode(value, outStream);
@@ -158,6 +160,7 @@ public void encode(T value, OutputStream outStream, Context context)
    * @throws CoderException if the value could not be decoded for some reason
    */
   @Deprecated
+  @Experimental(Kind.CODER_CONTEXT)
   public T decode(InputStream inStream, Context context)
       throws CoderException, IOException {
     return decode(inStream);

From 70680b5d830ed3518f417824068d1817c304174b Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 16:06:30 -0700
Subject: [PATCH 136/387] Don't log full exception traceback in direct runner.

It will be logged later if not caught.

This reduces duplication for failing tests, and noise for tests
expecting failure.
---
 sdks/python/apache_beam/runners/direct/executor.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sdks/python/apache_beam/runners/direct/executor.py b/sdks/python/apache_beam/runners/direct/executor.py
index 53aa3d0fb45d..0aa1aec42bd7 100644
--- a/sdks/python/apache_beam/runners/direct/executor.py
+++ b/sdks/python/apache_beam/runners/direct/executor.py
@@ -24,7 +24,6 @@
 import Queue
 import sys
 import threading
-import traceback
 from weakref import WeakValueDictionary
 
 from apache_beam.metrics.execution import MetricsContainer
@@ -308,7 +307,6 @@ def call(self):
       self._completion_callback.handle_result(self._input_bundle, result)
       return result
     except Exception as e:  # pylint: disable=broad-except
-      logging.warning('Task failed: %s', traceback.format_exc(), exc_info=True)
       self._completion_callback.handle_exception(e)
     finally:
       self._evaluation_context.metrics().commit_physical(

From dc186fd876c5253d410802c9161d669124f0706f Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 15:54:23 -0700
Subject: [PATCH 137/387] [BEAM-2184] Rename OutputTimeFn to TimestampCombiner.

---
 sdks/python/apache_beam/transforms/core.py    | 23 +++++++-------
 .../python/apache_beam/transforms/timeutil.py | 22 +++++++-------
 sdks/python/apache_beam/transforms/trigger.py | 26 ++++++++--------
 .../apache_beam/transforms/trigger_test.py    |  9 +++---
 .../transforms/trigger_transcripts.yaml       | 30 +++++++++----------
 sdks/python/apache_beam/transforms/window.py  | 22 +++++++-------
 .../apache_beam/transforms/window_test.py     |  4 +--
 7 files changed, 69 insertions(+), 67 deletions(-)

diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index 9367e6f6ff9c..7ca1632c08e7 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -33,7 +33,7 @@
 from apache_beam.transforms.ptransform import PTransform
 from apache_beam.transforms.ptransform import PTransformWithSideInputs
 from apache_beam.transforms.window import MIN_TIMESTAMP
-from apache_beam.transforms.window import OutputTimeFn
+from apache_beam.transforms.window import TimestampCombiner
 from apache_beam.transforms.window import WindowedValue
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.transforms.window import GlobalWindows
@@ -1172,7 +1172,7 @@ def expand(self, pcoll):
 class Windowing(object):
 
   def __init__(self, windowfn, triggerfn=None, accumulation_mode=None,
-               output_time_fn=None):
+               timestamp_combiner=None):
     global AccumulationMode, DefaultTrigger  # pylint: disable=global-variable-not-assigned
     # pylint: disable=wrong-import-order, wrong-import-position
     from apache_beam.transforms.trigger import AccumulationMode, DefaultTrigger
@@ -1192,17 +1192,18 @@ def __init__(self, windowfn, triggerfn=None, accumulation_mode=None,
     self.windowfn = windowfn
     self.triggerfn = triggerfn
     self.accumulation_mode = accumulation_mode
-    self.output_time_fn = output_time_fn or OutputTimeFn.OUTPUT_AT_EOW
+    self.timestamp_combiner = (
+        timestamp_combiner or TimestampCombiner.OUTPUT_AT_EOW)
     self._is_default = (
         self.windowfn == GlobalWindows() and
         self.triggerfn == DefaultTrigger() and
         self.accumulation_mode == AccumulationMode.DISCARDING and
-        self.output_time_fn == OutputTimeFn.OUTPUT_AT_EOW)
+        self.timestamp_combiner == TimestampCombiner.OUTPUT_AT_EOW)
 
   def __repr__(self):
     return "Windowing(%s, %s, %s, %s)" % (self.windowfn, self.triggerfn,
                                           self.accumulation_mode,
-                                          self.output_time_fn)
+                                          self.timestamp_combiner)
 
   def __eq__(self, other):
     if type(self) == type(other):
@@ -1212,7 +1213,7 @@ def __eq__(self, other):
           self.windowfn == other.windowfn
           and self.triggerfn == other.triggerfn
           and self.accumulation_mode == other.accumulation_mode
-          and self.output_time_fn == other.output_time_fn)
+          and self.timestamp_combiner == other.timestamp_combiner)
     return False
 
   def is_default(self):
@@ -1229,7 +1230,7 @@ def to_runner_api(self, context):
             self.windowfn.get_window_coder()),
         trigger=self.triggerfn.to_runner_api(context),
         accumulation_mode=self.accumulation_mode,
-        output_time=self.output_time_fn,
+        output_time=self.timestamp_combiner,
         # TODO(robertwb): Support EMIT_IF_NONEMPTY
         closing_behavior=beam_runner_api_pb2.EMIT_ALWAYS,
         allowed_lateness=0)
@@ -1242,7 +1243,7 @@ def from_runner_api(proto, context):
         windowfn=WindowFn.from_runner_api(proto.window_fn, context),
         triggerfn=TriggerFn.from_runner_api(proto.trigger, context),
         accumulation_mode=proto.accumulation_mode,
-        output_time_fn=proto.output_time)
+        timestamp_combiner=proto.output_time)
 
 
 @typehints.with_input_types(T)
@@ -1275,9 +1276,9 @@ def __init__(self, windowfn, **kwargs):
     """
     triggerfn = kwargs.pop('trigger', None)
     accumulation_mode = kwargs.pop('accumulation_mode', None)
-    output_time_fn = kwargs.pop('output_time_fn', None)
+    timestamp_combiner = kwargs.pop('timestamp_combiner', None)
     self.windowing = Windowing(windowfn, triggerfn, accumulation_mode,
-                               output_time_fn)
+                               timestamp_combiner)
     super(WindowInto, self).__init__(self.WindowIntoFn(self.windowing))
 
   def get_windowing(self, unused_inputs):
@@ -1307,7 +1308,7 @@ def from_runner_api_parameter(proto, context):
         windowing.windowfn,
         trigger=windowing.triggerfn,
         accumulation_mode=windowing.accumulation_mode,
-        output_time_fn=windowing.output_time_fn)
+        timestamp_combiner=windowing.timestamp_combiner)
 
 
 PTransform.register_urn(
diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py
index 5453b20c4aad..435d41b65e85 100644
--- a/sdks/python/apache_beam/transforms/timeutil.py
+++ b/sdks/python/apache_beam/transforms/timeutil.py
@@ -49,8 +49,8 @@ def from_string(domain):
     raise ValueError('Unknown time domain: %s' % domain)
 
 
-class OutputTimeFnImpl(object):
-  """Implementation of OutputTimeFn."""
+class TimestampCombinerImpl(object):
+  """Implementation of TimestampCombiner."""
 
   __metaclass__ = ABCMeta
 
@@ -78,8 +78,8 @@ def merge(self, unused_result_window, merging_timestamps):
     return self.combine_all(merging_timestamps)
 
 
-class DependsOnlyOnWindow(OutputTimeFnImpl):
-  """OutputTimeFnImpl that only depends on the window."""
+class DependsOnlyOnWindow(TimestampCombinerImpl):
+  """TimestampCombinerImpl that only depends on the window."""
 
   __metaclass__ = ABCMeta
 
@@ -92,8 +92,8 @@ def merge(self, result_window, unused_merging_timestamps):
     return self.assign_output_time(result_window, None)
 
 
-class OutputAtEarliestInputTimestampImpl(OutputTimeFnImpl):
-  """OutputTimeFnImpl outputting at earliest input timestamp."""
+class OutputAtEarliestInputTimestampImpl(TimestampCombinerImpl):
+  """TimestampCombinerImpl outputting at earliest input timestamp."""
 
   def assign_output_time(self, window, input_timestamp):
     return input_timestamp
@@ -103,8 +103,8 @@ def combine(self, output_timestamp, other_output_timestamp):
     return min(output_timestamp, other_output_timestamp)
 
 
-class OutputAtEarliestTransformedInputTimestampImpl(OutputTimeFnImpl):
-  """OutputTimeFnImpl outputting at earliest input timestamp."""
+class OutputAtEarliestTransformedInputTimestampImpl(TimestampCombinerImpl):
+  """TimestampCombinerImpl outputting at earliest input timestamp."""
 
   def __init__(self, window_fn):
     self.window_fn = window_fn
@@ -116,8 +116,8 @@ def combine(self, output_timestamp, other_output_timestamp):
     return min(output_timestamp, other_output_timestamp)
 
 
-class OutputAtLatestInputTimestampImpl(OutputTimeFnImpl):
-  """OutputTimeFnImpl outputting at latest input timestamp."""
+class OutputAtLatestInputTimestampImpl(TimestampCombinerImpl):
+  """TimestampCombinerImpl outputting at latest input timestamp."""
 
   def assign_output_time(self, window, input_timestamp):
     return input_timestamp
@@ -127,7 +127,7 @@ def combine(self, output_timestamp, other_output_timestamp):
 
 
 class OutputAtEndOfWindowImpl(DependsOnlyOnWindow):
-  """OutputTimeFnImpl outputting at end of window."""
+  """TimestampCombinerImpl outputting at end of window."""
 
   def assign_output_time(self, window, unused_input_timestamp):
     return window.end
diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py
index b9786f4b41d5..bcb9dd307de1 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -32,7 +32,7 @@
 from apache_beam.transforms.timeutil import MIN_TIMESTAMP
 from apache_beam.transforms.timeutil import TimeDomain
 from apache_beam.transforms.window import GlobalWindow
-from apache_beam.transforms.window import OutputTimeFn
+from apache_beam.transforms.window import TimestampCombiner
 from apache_beam.transforms.window import WindowedValue
 from apache_beam.transforms.window import WindowFn
 from apache_beam.runners.api import beam_runner_api_pb2
@@ -100,17 +100,17 @@ def with_prefix(self, prefix):
 
 class WatermarkHoldStateTag(StateTag):
 
-  def __init__(self, tag, output_time_fn_impl):
+  def __init__(self, tag, timestamp_combiner_impl):
     super(WatermarkHoldStateTag, self).__init__(tag)
-    self.output_time_fn_impl = output_time_fn_impl
+    self.timestamp_combiner_impl = timestamp_combiner_impl
 
   def __repr__(self):
     return 'WatermarkHoldStateTag(%s, %s)' % (self.tag,
-                                              self.output_time_fn_impl)
+                                              self.timestamp_combiner_impl)
 
   def with_prefix(self, prefix):
     return WatermarkHoldStateTag(prefix + self.tag,
-                                 self.output_time_fn_impl)
+                                 self.timestamp_combiner_impl)
 
 
 # pylint: disable=unused-argument
@@ -750,7 +750,7 @@ def get_state(self, window, tag):
     elif isinstance(tag, ListStateTag):
       return [v for vs in values for v in vs]
     elif isinstance(tag, WatermarkHoldStateTag):
-      return tag.output_time_fn_impl.combine_all(values)
+      return tag.timestamp_combiner_impl.combine_all(values)
     else:
       raise ValueError('Invalid tag.', tag)
 
@@ -909,11 +909,11 @@ class GeneralTriggerDriver(TriggerDriver):
 
   def __init__(self, windowing):
     self.window_fn = windowing.windowfn
-    self.output_time_fn_impl = OutputTimeFn.get_impl(windowing.output_time_fn,
-                                                     self.window_fn)
+    self.timestamp_combiner_impl = TimestampCombiner.get_impl(
+        windowing.timestamp_combiner, self.window_fn)
     # pylint: disable=invalid-name
-    self.WATERMARK_HOLD = WatermarkHoldStateTag('watermark',
-                                                self.output_time_fn_impl)
+    self.WATERMARK_HOLD = WatermarkHoldStateTag(
+        'watermark', self.timestamp_combiner_impl)
     # pylint: enable=invalid-name
     self.trigger_fn = windowing.triggerfn
     self.accumulation_mode = windowing.accumulation_mode
@@ -965,10 +965,10 @@ def merge(_, to_be_merged, merge_result):  # pylint: disable=no-self-argument
         continue
       # Add watermark hold.
       # TODO(ccy): Add late data and garbage-collection hold support.
-      output_time = self.output_time_fn_impl.merge(
+      output_time = self.timestamp_combiner_impl.merge(
           window,
           (element_output_time for element_output_time in
-           (self.output_time_fn_impl.assign_output_time(window, timestamp)
+           (self.timestamp_combiner_impl.assign_output_time(window, timestamp)
             for unused_value, timestamp in elements)
            if element_output_time >= output_watermark))
       if output_time is not None:
@@ -1075,7 +1075,7 @@ def get_state(self, window, tag):
     elif isinstance(tag, ListStateTag):
       return values
     elif isinstance(tag, WatermarkHoldStateTag):
-      return tag.output_time_fn_impl.combine_all(values)
+      return tag.timestamp_combiner_impl.combine_all(values)
     else:
       raise ValueError('Invalid tag.', tag)
 
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index 914babbb0968..38871fed3abd 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -44,7 +44,7 @@
 from apache_beam.transforms.window import FixedWindows
 from apache_beam.transforms.window import IntervalWindow
 from apache_beam.transforms.window import MIN_TIMESTAMP
-from apache_beam.transforms.window import OutputTimeFn
+from apache_beam.transforms.window import TimestampCombiner
 from apache_beam.transforms.window import Sessions
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.transforms.window import WindowedValue
@@ -522,11 +522,12 @@ def parse_fn(s, names):
     trigger_fn = parse_fn(spec.get('trigger_fn', 'Default'), trigger_names)
     accumulation_mode = getattr(
         AccumulationMode, spec.get('accumulation_mode', 'ACCUMULATING').upper())
-    output_time_fn = getattr(
-        OutputTimeFn, spec.get('output_time_fn', 'OUTPUT_AT_EOW').upper())
+    timestamp_combiner = getattr(
+        TimestampCombiner,
+        spec.get('timestamp_combiner', 'OUTPUT_AT_EOW').upper())
 
     driver = GeneralTriggerDriver(
-        Windowing(window_fn, trigger_fn, accumulation_mode, output_time_fn))
+        Windowing(window_fn, trigger_fn, accumulation_mode, timestamp_combiner))
     state = InMemoryUnmergedState()
     output = []
     watermark = MIN_TIMESTAMP
diff --git a/sdks/python/apache_beam/transforms/trigger_transcripts.yaml b/sdks/python/apache_beam/transforms/trigger_transcripts.yaml
index f87cd1dc767f..a736e943ac59 100644
--- a/sdks/python/apache_beam/transforms/trigger_transcripts.yaml
+++ b/sdks/python/apache_beam/transforms/trigger_transcripts.yaml
@@ -29,7 +29,7 @@ transcript:                       # Ordered list of events.
 name: fixed_default_late_data
 window_fn: FixedWindows(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_EOW
+timestamp_combiner: OUTPUT_AT_EOW
 transcript:
   - input: [1, 2, 3, 10, 11, 25]
   - watermark: 100
@@ -42,10 +42,10 @@ transcript:
       - {window: [0, 9], values: [1, 2, 3, 7], timestamp: 10, late: true}
 
 ---
-name: output_time_fn_earliest
+name: timestamp_combiner_earliest
 window_fn: FixedWindows(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_EARLIEST
+timestamp_combiner: OUTPUT_AT_EARLIEST
 transcript:
   - input: [1, 2, 3, 10, 11, 25]
   - watermark: 100
@@ -55,10 +55,10 @@ transcript:
       - {window: [20, 29], values: [25], timestamp: 25, late: false}
 
 ---
-name: output_time_fn_latest
+name: timestamp_combiner_latest
 window_fn: FixedWindows(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_LATEST
+timestamp_combiner: OUTPUT_AT_LATEST
 transcript:
   - input: [1, 2, 3, 10, 11, 25]
   - watermark: 100
@@ -69,10 +69,10 @@ transcript:
 
 ---
 # Test that custom timestamping is not invoked.
-name: output_time_fn_custom_timestamping_eow
+name: timestamp_combiner_custom_timestamping_eow
 window_fn: CustomTimestampingFixedWindowsWindowFn(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_EOW
+timestamp_combiner: OUTPUT_AT_EOW
 transcript:
   - input: [1, 2, 3, 10, 11, 25]
   - watermark: 100
@@ -83,10 +83,10 @@ transcript:
 
 ---
 # Test that custom timestamping is not invoked.
-name: output_time_fn_custom_timestamping_earliest
+name: timestamp_combiner_custom_timestamping_earliest
 window_fn: CustomTimestampingFixedWindowsWindowFn(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_EARLIEST
+timestamp_combiner: OUTPUT_AT_EARLIEST
 transcript:
   - input: [1, 2, 3, 10, 11, 25]
   - watermark: 100
@@ -97,10 +97,10 @@ transcript:
 
 ---
 # Test that custom timestamping is in fact invoked.
-name: output_time_fn_custom_timestamping_earliest
+name: timestamp_combiner_custom_timestamping_earliest
 window_fn: CustomTimestampingFixedWindowsWindowFn(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_EARLIEST_TRANSFORMED
+timestamp_combiner: OUTPUT_AT_EARLIEST_TRANSFORMED
 transcript:
   - input: [1, 2, 3, 10, 11, 25]
   - watermark: 100
@@ -113,7 +113,7 @@ transcript:
 name: early_late_sessions
 window_fn: Sessions(10)
 trigger_fn: AfterWatermark(early=AfterCount(2), late=AfterCount(3))
-output_time_fn: OUTPUT_AT_EOW
+timestamp_combiner: OUTPUT_AT_EOW
 transcript:
     - input: [1, 2, 3]
     - expect:
@@ -136,7 +136,7 @@ transcript:
 name: garbage_collection
 window_fn: FixedWindows(10)
 trigger_fn: AfterCount(2)
-output_time_fn: OUTPUT_AT_EOW
+timestamp_combiner: OUTPUT_AT_EOW
 allowed_lateness: 10
 accumulation_mode: discarding
 transcript:
@@ -153,7 +153,7 @@ transcript:
 name: known_late_data_watermark
 window_fn: FixedWindows(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_EARLIEST
+timestamp_combiner: OUTPUT_AT_EARLIEST
 transcript:
   - watermark: 5
   - input: [2, 3, 7, 8]
@@ -165,7 +165,7 @@ transcript:
 name: known_late_data_no_watermark_hold_possible
 window_fn: FixedWindows(10)
 trigger_fn: Default
-output_time_fn: OUTPUT_AT_EARLIEST
+timestamp_combiner: OUTPUT_AT_EARLIEST
 transcript:
   - watermark: 8
   - input: [2, 3, 7]
diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py
index 9c4b1090c894..44a5a267d8dc 100644
--- a/sdks/python/apache_beam/transforms/window.py
+++ b/sdks/python/apache_beam/transforms/window.py
@@ -67,7 +67,7 @@
 
 # TODO(ccy): revisit naming and semantics once Java Apache Beam finalizes their
 # behavior.
-class OutputTimeFn(object):
+class TimestampCombiner(object):
   """Determines how output timestamps of grouping operations are assigned."""
 
   OUTPUT_AT_EOW = beam_runner_api_pb2.END_OF_WINDOW
@@ -77,17 +77,17 @@ class OutputTimeFn(object):
   OUTPUT_AT_EARLIEST_TRANSFORMED = 'OUTPUT_AT_EARLIEST_TRANSFORMED'
 
   @staticmethod
-  def get_impl(output_time_fn, window_fn):
-    if output_time_fn == OutputTimeFn.OUTPUT_AT_EOW:
+  def get_impl(timestamp_combiner, window_fn):
+    if timestamp_combiner == TimestampCombiner.OUTPUT_AT_EOW:
       return timeutil.OutputAtEndOfWindowImpl()
-    elif output_time_fn == OutputTimeFn.OUTPUT_AT_EARLIEST:
+    elif timestamp_combiner == TimestampCombiner.OUTPUT_AT_EARLIEST:
       return timeutil.OutputAtEarliestInputTimestampImpl()
-    elif output_time_fn == OutputTimeFn.OUTPUT_AT_LATEST:
+    elif timestamp_combiner == TimestampCombiner.OUTPUT_AT_LATEST:
       return timeutil.OutputAtLatestInputTimestampImpl()
-    elif output_time_fn == OutputTimeFn.OUTPUT_AT_EARLIEST_TRANSFORMED:
+    elif timestamp_combiner == TimestampCombiner.OUTPUT_AT_EARLIEST_TRANSFORMED:
       return timeutil.OutputAtEarliestTransformedInputTimestampImpl(window_fn)
     else:
-      raise ValueError('Invalid OutputTimeFn: %s.' % output_time_fn)
+      raise ValueError('Invalid TimestampCombiner: %s.' % timestamp_combiner)
 
 
 class WindowFn(urns.RunnerApiFn):
@@ -132,10 +132,10 @@ def get_window_coder(self):
   def get_transformed_output_time(self, window, input_timestamp):  # pylint: disable=unused-argument
     """Given input time and output window, returns output time for window.
 
-    If OutputTimeFn.OUTPUT_AT_EARLIEST_TRANSFORMED is used in the Windowing,
-    the output timestamp for the given window will be the earliest of the
-    timestamps returned by get_transformed_output_time() for elements of the
-    window.
+    If TimestampCombiner.OUTPUT_AT_EARLIEST_TRANSFORMED is used in the
+    Windowing, the output timestamp for the given window will be the earliest
+    of the timestamps returned by get_transformed_output_time() for elements
+    of the window.
 
     Arguments:
       window: Output window of element.
diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py
index 38a2df8c31e6..0f613d788413 100644
--- a/sdks/python/apache_beam/transforms/window_test.py
+++ b/sdks/python/apache_beam/transforms/window_test.py
@@ -38,7 +38,7 @@
 from apache_beam.transforms.window import GlobalWindow
 from apache_beam.transforms.window import GlobalWindows
 from apache_beam.transforms.window import IntervalWindow
-from apache_beam.transforms.window import OutputTimeFn
+from apache_beam.transforms.window import TimestampCombiner
 from apache_beam.transforms.window import Sessions
 from apache_beam.transforms.window import SlidingWindows
 from apache_beam.transforms.window import TimestampedValue
@@ -271,7 +271,7 @@ def test_windowing_encoding(self):
         Windowing(FixedWindows(1, 3), AfterCount(6),
                   accumulation_mode=AccumulationMode.ACCUMULATING),
         Windowing(SlidingWindows(10, 15, 21), AfterCount(28),
-                  output_time_fn=OutputTimeFn.OUTPUT_AT_LATEST,
+                  timestamp_combiner=TimestampCombiner.OUTPUT_AT_LATEST,
                   accumulation_mode=AccumulationMode.DISCARDING)):
       context = pipeline_context.PipelineContext()
       self.assertEqual(

From 9283a5e8f931f554b507739f4448c862caa7e5cd Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 8 May 2017 13:33:15 -0700
Subject: [PATCH 138/387] [BEAM-1925] validate DoFn at pipeline creation time

---
 sdks/python/apache_beam/runners/common.py     | 25 ++++----
 .../python/apache_beam/runners/common_test.py | 58 +++++++++++++++++++
 sdks/python/apache_beam/transforms/core.py    |  9 ++-
 .../apache_beam/transforms/ptransform_test.py |  1 -
 4 files changed, 78 insertions(+), 15 deletions(-)
 create mode 100644 sdks/python/apache_beam/runners/common_test.py

diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py
index 045c1093836f..74c61abc34bc 100644
--- a/sdks/python/apache_beam/runners/common.py
+++ b/sdks/python/apache_beam/runners/common.py
@@ -95,17 +95,20 @@ def __init__(self, do_fn):
     self._validate()
 
   def _validate(self):
+    self._validate_process()
     self._validate_bundle_method(self.start_bundle_method)
     self._validate_bundle_method(self.finish_bundle_method)
 
-  def _validate_bundle_method(self, method_wrapper):
-    # Here we use the fact that every DoFn parameter defined in core.DoFn has
-    # the value that is the same as the name of the parameter and ends with
-    # string 'Param'.
-    unsupported_dofn_params = [i for i in core.DoFn.__dict__ if
-                               i.endswith('Param')]
+  def _validate_process(self):
+    """Validate that none of the DoFnParameters are repeated in the function
+    """
+    for param in core.DoFn.DoFnParams:
+      assert self.process_method.defaults.count(param) <= 1
 
-    for param in unsupported_dofn_params:
+  def _validate_bundle_method(self, method_wrapper):
+    """Validate that none of the DoFnParameters are used in the function
+    """
+    for param in core.DoFn.DoFnParams:
       assert param not in method_wrapper.defaults
 
 
@@ -156,18 +159,14 @@ def invoke_process(self, windowed_value):
   def invoke_start_bundle(self):
     """Invokes the DoFn.start_bundle() method.
     """
-    args_for_start_bundle = self.signature.start_bundle_method.defaults
     self.output_processor.start_bundle_outputs(
-        self.signature.start_bundle_method.method_value(
-            *args_for_start_bundle))
+        self.signature.start_bundle_method.method_value())
 
   def invoke_finish_bundle(self):
     """Invokes the DoFn.finish_bundle() method.
     """
-    args_for_finish_bundle = self.signature.finish_bundle_method.defaults
     self.output_processor.finish_bundle_outputs(
-        self.signature.finish_bundle_method.method_value(
-            *args_for_finish_bundle))
+        self.signature.finish_bundle_method.method_value())
 
 
 class SimpleInvoker(DoFnInvoker):
diff --git a/sdks/python/apache_beam/runners/common_test.py b/sdks/python/apache_beam/runners/common_test.py
new file mode 100644
index 000000000000..62a6955f6ce4
--- /dev/null
+++ b/sdks/python/apache_beam/runners/common_test.py
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+from apache_beam.transforms.core import DoFn
+from apache_beam.runners.common import DoFnSignature
+
+
+class DoFnSignatureTest(unittest.TestCase):
+
+  def test_dofn_validate_process_error(self):
+    class MyDoFn(DoFn):
+      def process(self, element, w1=DoFn.WindowParam, w2=DoFn.WindowParam):
+        pass
+
+    with self.assertRaises(AssertionError):
+      DoFnSignature(MyDoFn())
+
+  def test_dofn_validate_start_bundle_error(self):
+    class MyDoFn(DoFn):
+      def process(self, element):
+        pass
+
+      def start_bundle(self, w1=DoFn.WindowParam):
+        pass
+
+    with self.assertRaises(AssertionError):
+      DoFnSignature(MyDoFn())
+
+  def test_dofn_validate_finish_bundle_error(self):
+    class MyDoFn(DoFn):
+      def process(self, element):
+        pass
+
+      def finish_bundle(self, w1=DoFn.WindowParam):
+        pass
+
+    with self.assertRaises(AssertionError):
+      DoFnSignature(MyDoFn())
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index 7ca1632c08e7..e37a387e6d18 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -29,7 +29,8 @@
 from apache_beam.internal import util
 from apache_beam.runners.api import beam_runner_api_pb2
 from apache_beam.transforms import ptransform
-from apache_beam.transforms.display import HasDisplayData, DisplayDataItem
+from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.transforms.display import HasDisplayData
 from apache_beam.transforms.ptransform import PTransform
 from apache_beam.transforms.ptransform import PTransformWithSideInputs
 from apache_beam.transforms.window import MIN_TIMESTAMP
@@ -131,6 +132,8 @@ class DoFn(WithTypeHints, HasDisplayData):
   TimestampParam = 'TimestampParam'
   WindowParam = 'WindowParam'
 
+  DoFnParams = [ElementParam, SideInputParam, TimestampParam, WindowParam]
+
   @staticmethod
   def from_callable(fn):
     return CallableWrapperDoFn(fn)
@@ -596,6 +599,10 @@ def __init__(self, fn, *args, **kwargs):
     if not isinstance(self.fn, DoFn):
       raise TypeError('ParDo must be called with a DoFn instance.')
 
+    # Validate the DoFn by creating a DoFnSignature
+    from apache_beam.runners.common import DoFnSignature
+    DoFnSignature(self.fn)
+
   def default_type_hints(self):
     return self.fn.get_type_hints()
 
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index b8b0733622e4..e7126610935b 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -303,7 +303,6 @@ def __init__(self):
 
       def start_bundle(self):
         self.state = 'started'
-        return None
 
       def process(self, element):
         if self.state == 'started':

From 81b89acb46d79dabc9c21d8421868c79a609a9dc Mon Sep 17 00:00:00 2001
From: Stas Levin <staslevin@apache.org>
Date: Mon, 1 May 2017 07:30:49 +0300
Subject: [PATCH 139/387] [BEAM-2095] Made SourceRDD hasNext idempotent

---
 .../beam/runners/spark/io/SourceRDD.java      | 173 ++++++++++++------
 .../spark/io/ReaderToIteratorAdapterTest.java | 145 +++++++++++++++
 2 files changed, 260 insertions(+), 58 deletions(-)
 create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
index 71a19e796d4d..aa89c59fc4b0 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
@@ -19,12 +19,15 @@
 package org.apache.beam.runners.spark.io;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.NoSuchElementException;
 import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
 import org.apache.beam.sdk.io.BoundedSource;
@@ -47,6 +50,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import scala.Option;
+import scala.collection.JavaConversions;
 
 /**
  * Classes implementing Beam {@link Source} {@link RDD}s.
@@ -118,80 +122,133 @@ public Partition[] getPartitions() {
       }
     }
 
+    private BoundedSource.BoundedReader<T> createReader(SourcePartition<T> partition) {
+      try {
+        return ((BoundedSource<T>) partition.source).createReader(
+            runtimeContext.getPipelineOptions());
+      } catch (IOException e) {
+        throw new RuntimeException("Failed to create reader from a BoundedSource.", e);
+      }
+    }
+
     @Override
     public scala.collection.Iterator<WindowedValue<T>> compute(final Partition split,
-                                                               TaskContext context) {
+                                                               final TaskContext context) {
       final MetricsContainer metricsContainer = metricsAccum.localValue().getContainer(stepName);
 
-      final Iterator<WindowedValue<T>> iter = new Iterator<WindowedValue<T>>() {
-        @SuppressWarnings("unchecked")
-        SourcePartition<T> partition = (SourcePartition<T>) split;
-        BoundedSource.BoundedReader<T> reader = createReader(partition);
-
-        private boolean finished = false;
-        private boolean started = false;
-        private boolean closed = false;
-
-        @Override
-        public boolean hasNext() {
-          // Add metrics container to the scope of org.apache.beam.sdk.io.Source.Reader methods
-          // since they may report metrics.
-          try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
-            try {
-              if (!started) {
-                started = true;
-                finished = !reader.start();
-              } else {
-                finished = !reader.advance();
-              }
-              if (finished) {
-                // safely close the reader if there are no more elements left to read.
-                closeIfNotClosed();
-              }
-              return !finished;
-            } catch (IOException e) {
-              closeIfNotClosed();
-              throw new RuntimeException("Failed to read from reader.", e);
+      @SuppressWarnings("unchecked")
+      final BoundedSource.BoundedReader<T> reader = createReader((SourcePartition<T>) split);
+
+      final Iterator<WindowedValue<T>> readerIterator =
+          new ReaderToIteratorAdapter<>(metricsContainer, reader);
+
+      return new InterruptibleIterator<>(context, JavaConversions.asScalaIterator(readerIterator));
+    }
+
+    /**
+     * Exposes an <code>Iterator</code>&lt;{@link WindowedValue}&gt; interface on top of a
+     * {@link Source.Reader}.
+     * <p>
+     *   <code>hasNext</code> is idempotent and returns <code>true</code> iff further items are
+     *   available for reading using the underlying reader.
+     *   Consequently, when the reader is closed, or when the reader has no further elements
+     *   available (i.e, {@link Source.Reader#advance()} returned <code>false</code>),
+     *   <code>hasNext</code> returns <code>false</code>.
+     * </p>
+     * <p>
+     *   Since this is a read-only iterator, an attempt to call <code>remove</code> will throw an
+     *   <code>UnsupportedOperationException</code>.
+     * </p>
+     */
+    @VisibleForTesting
+    static class ReaderToIteratorAdapter<T> implements Iterator<WindowedValue<T>> {
+
+      private static final boolean FAILED_TO_OBTAIN_NEXT = false;
+      private static final boolean SUCCESSFULLY_OBTAINED_NEXT = true;
+
+      private final MetricsContainer metricsContainer;
+      private final Source.Reader<T> reader;
+
+      private boolean started = false;
+      private boolean closed = false;
+      private WindowedValue<T> next = null;
+
+      ReaderToIteratorAdapter(final MetricsContainer metricsContainer,
+                              final Source.Reader<T> reader) {
+        this.metricsContainer = metricsContainer;
+        this.reader = reader;
+      }
+
+      private boolean tryProduceNext() {
+        try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
+          if (closed) {
+            return FAILED_TO_OBTAIN_NEXT;
+          } else {
+            checkState(next == null, "unexpected non-null value for next");
+            if (seekNext()) {
+              next = WindowedValue.timestampedValueInGlobalWindow(reader.getCurrent(),
+                                                                  reader.getCurrentTimestamp());
+              return SUCCESSFULLY_OBTAINED_NEXT;
+            } else {
+              close();
+              return FAILED_TO_OBTAIN_NEXT;
             }
-          } catch (IOException e) {
-            throw new RuntimeException(e);
           }
+        } catch (final Exception e) {
+          throw new RuntimeException("Failed to read data.", e);
+        }
+      }
+
+      private void close() {
+        closed = true;
+        try {
+          reader.close();
+        } catch (final IOException e) {
+          throw new RuntimeException(e);
         }
+      }
 
-        @Override
-        public WindowedValue<T> next() {
-          return WindowedValue.timestampedValueInGlobalWindow(reader.getCurrent(),
-              reader.getCurrentTimestamp());
+      private boolean seekNext() throws IOException {
+        if (!started) {
+          started = true;
+          return reader.start();
+        } else {
+          return !closed && reader.advance();
         }
+      }
 
-        @Override
-        public void remove() {
-          throw new UnsupportedOperationException("Remove from partition iterator is not allowed.");
+      private WindowedValue<T> consumeCurrent() {
+        if (next == null) {
+          throw new NoSuchElementException();
+        } else {
+          final WindowedValue<T> current = next;
+          next = null;
+          return current;
         }
+      }
 
-        private void closeIfNotClosed() {
-          if (!closed) {
-            closed = true;
-            try {
-              reader.close();
-            } catch (IOException e) {
-              throw new RuntimeException("Failed to close Reader.", e);
-            }
-          }
+      private WindowedValue<T> consumeNext() {
+        if (next == null) {
+          tryProduceNext();
         }
-      };
+        return consumeCurrent();
+      }
 
-      return new InterruptibleIterator<>(context,
-          scala.collection.JavaConversions.asScalaIterator(iter));
-    }
+      @Override
+      public boolean hasNext() {
+        return next != null || tryProduceNext();
+      }
 
-    private BoundedSource.BoundedReader<T> createReader(SourcePartition<T> partition) {
-      try {
-        return ((BoundedSource<T>) partition.source).createReader(
-            runtimeContext.getPipelineOptions());
-      } catch (IOException e) {
-        throw new RuntimeException("Failed to create reader from a BoundedSource.", e);
+      @Override
+      public WindowedValue<T> next() {
+        return consumeNext();
+      }
+
+      @Override
+      public void remove() {
+        throw new UnsupportedOperationException();
       }
+
     }
   }
 
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java
new file mode 100644
index 000000000000..86f35ba09559
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.io;
+
+import static com.google.common.base.Preconditions.checkState;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.metrics.MetricsContainer;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Test for {@link SourceRDD.Bounded.ReaderToIteratorAdapter}.
+ */
+public class ReaderToIteratorAdapterTest {
+
+  @Rule
+  public ExpectedException exception = ExpectedException.none();
+
+  private static class TestReader extends Source.Reader<Integer> {
+
+    static final int LIMIT = 4;
+    static final int START = 1;
+
+    private Integer current = START - 1;
+    private boolean closed = false;
+    private boolean drained = false;
+
+    boolean isClosed() {
+      return closed;
+    }
+
+    @Override
+    public boolean start() throws IOException {
+      return advance();
+    }
+
+    @Override
+    public boolean advance() throws IOException {
+      checkState(!drained && !closed);
+      drained = ++current >= LIMIT;
+      return !drained;
+    }
+
+    @Override
+    public Integer getCurrent() throws NoSuchElementException {
+      checkState(!drained && !closed);
+      return current;
+    }
+
+    @Override
+    public Instant getCurrentTimestamp() throws NoSuchElementException {
+      checkState(!drained && !closed);
+      return Instant.now();
+    }
+
+    @Override
+    public void close() throws IOException {
+      checkState(!closed);
+      closed = true;
+    }
+
+    @Override
+    public Source<Integer> getCurrentSource() {
+      return null;
+    }
+  }
+
+  private final TestReader testReader = new TestReader();
+
+  private final SourceRDD.Bounded.ReaderToIteratorAdapter<Integer> readerIterator =
+      new SourceRDD.Bounded.ReaderToIteratorAdapter<>(new MetricsContainer(""), testReader);
+
+  private void assertReaderRange(final int start, final int end) {
+    for (int i = start; i < end; i++) {
+      assertThat(readerIterator.next().getValue(), is(i));
+    }
+  }
+
+  @Test
+  public void testReaderIsClosedAfterDrainage() throws Exception {
+    assertReaderRange(TestReader.START, TestReader.LIMIT);
+
+    assertThat(readerIterator.hasNext(), is(false));
+
+    // reader is closed only after hasNext realises there are no more elements
+    assertThat(testReader.isClosed(), is(true));
+  }
+
+  @Test
+  public void testNextWhenDrainedThrows() throws Exception {
+    assertReaderRange(TestReader.START, TestReader.LIMIT);
+
+    exception.expect(NoSuchElementException.class);
+    readerIterator.next();
+  }
+
+  @Test
+  public void testHasNextIdempotencyCombo() throws Exception {
+    assertThat(readerIterator.hasNext(), is(true));
+    assertThat(readerIterator.hasNext(), is(true));
+
+    assertThat(readerIterator.next().getValue(), is(1));
+
+    assertThat(readerIterator.hasNext(), is(true));
+    assertThat(readerIterator.hasNext(), is(true));
+    assertThat(readerIterator.hasNext(), is(true));
+
+    assertThat(readerIterator.next().getValue(), is(2));
+    assertThat(readerIterator.next().getValue(), is(3));
+
+    // drained
+
+    assertThat(readerIterator.hasNext(), is(false));
+    assertThat(readerIterator.hasNext(), is(false));
+
+    // no next to give
+
+    exception.expect(NoSuchElementException.class);
+    readerIterator.next();
+  }
+
+}

From 1f1940577bde19cf46c99deafb09687f331822bc Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Mon, 8 May 2017 16:18:20 -0700
Subject: [PATCH 140/387] Actually run after-count tests.

---
 .../apache_beam/transforms/trigger_test.py    | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index 38871fed3abd..c08a791bab4e 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -402,22 +402,22 @@ def test_trigger_encoding(self):
 class TriggerPipelineTest(unittest.TestCase):
 
   def test_after_count(self):
-    p = TestPipeline()
-    result = (p
-              | beam.Create([1, 2, 3, 4, 5, 10, 11])
-              | beam.FlatMap(lambda t: [('A', t), ('B', t + 5)])
-              | beam.Map(lambda (k, t): TimestampedValue((k, t), t))
-              | beam.WindowInto(FixedWindows(10), trigger=AfterCount(3),
-                                accumulation_mode=AccumulationMode.DISCARDING)
-              | beam.GroupByKey()
-              | beam.Map(lambda (k, v): ('%s-%s' % (k, len(v)), set(v))))
-    assert_that(result, equal_to(
-        {
-            'A-5': {1, 2, 3, 4, 5},
-            # A-10, A-11 never emitted due to AfterCount(3) never firing.
-            'B-4': {6, 7, 8, 9},
-            'B-3': {10, 15, 16},
-        }.iteritems()))
+    with TestPipeline() as p:
+      result = (p
+                | beam.Create([1, 2, 3, 4, 5, 10, 11])
+                | beam.FlatMap(lambda t: [('A', t), ('B', t + 5)])
+                | beam.Map(lambda (k, t): TimestampedValue((k, t), t))
+                | beam.WindowInto(FixedWindows(10), trigger=AfterCount(3),
+                                  accumulation_mode=AccumulationMode.DISCARDING)
+                | beam.GroupByKey()
+                | beam.Map(lambda (k, v): ('%s-%s' % (k, len(v)), set(v))))
+      assert_that(result, equal_to(
+          {
+              'A-5': {1, 2, 3, 4, 5},
+              # A-10, A-11 never emitted due to AfterCount(3) never firing.
+              'B-4': {6, 7, 8, 9},
+              'B-3': {10, 15, 16},
+          }.iteritems()))
 
 
 class TranscriptTest(unittest.TestCase):

From 3bf886056680ca50e11d36dc0da402bb0196a7c7 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 1 May 2017 15:00:24 -0700
Subject: [PATCH 141/387] [BEAM-1283] Finish bundle should only emit windowed
 values

---
 .../apache_beam/examples/snippets/snippets_test.py  | 10 ++++++++--
 sdks/python/apache_beam/io/iobase.py                |  4 +++-
 sdks/python/apache_beam/runners/common.py           | 13 ++-----------
 .../apache_beam/transforms/ptransform_test.py       |  8 +++++---
 4 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index 014809628ee1..da0a9625cfda 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -33,6 +33,7 @@
 from apache_beam.transforms.util import equal_to
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.examples.snippets import snippets
+from apache_beam.utils.windowed_value import WindowedValue
 
 # pylint: disable=expression-not-assigned
 from apache_beam.test_pipeline import TestPipeline
@@ -366,6 +367,7 @@ def parse_player_and_score(csv):
 
 class SnippetsTest(unittest.TestCase):
   # Replacing text read/write transforms with dummy transforms for testing.
+
   class DummyReadTransform(beam.PTransform):
     """A transform that will replace iobase.ReadFromText.
 
@@ -387,16 +389,20 @@ def process(self, element):
         pass
 
       def finish_bundle(self):
+        from apache_beam.transforms import window
+
         assert self.file_to_read
         for file_name in glob.glob(self.file_to_read):
           if self.compression_type is None:
             with open(file_name) as file:
               for record in file:
-                yield self.coder.decode(record.rstrip('\n'))
+                value = self.coder.decode(record.rstrip('\n'))
+                yield WindowedValue(value, -1, [window.GlobalWindow()])
           else:
             with gzip.open(file_name, 'r') as file:
               for record in file:
-                yield self.coder.decode(record.rstrip('\n'))
+                value = self.coder.decode(record.rstrip('\n'))
+                yield WindowedValue(value, -1, [window.GlobalWindow()])
 
     def expand(self, pcoll):
       return pcoll | beam.Create([None]) | 'DummyReadForTesting' >> beam.ParDo(
diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py
index 312542ad5d31..d47ef5b19cb3 100644
--- a/sdks/python/apache_beam/io/iobase.py
+++ b/sdks/python/apache_beam/io/iobase.py
@@ -44,6 +44,7 @@
 from apache_beam.transforms import window
 from apache_beam.transforms.display import HasDisplayData
 from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.utils.windowed_value import WindowedValue
 
 
 # Encapsulates information about a bundle of a source generated when method
@@ -931,7 +932,8 @@ def process(self, element, init_result):
 
   def finish_bundle(self):
     if self.writer is not None:
-      yield window.TimestampedValue(self.writer.close(), window.MAX_TIMESTAMP)
+      yield WindowedValue(self.writer.close(), window.MAX_TIMESTAMP,
+                          [window.GlobalWindow()])
 
 
 class _WriteKeyedBundleDoFn(core.DoFn):
diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py
index 74c61abc34bc..ec1f5dc47c77 100644
--- a/sdks/python/apache_beam/runners/common.py
+++ b/sdks/python/apache_beam/runners/common.py
@@ -487,18 +487,9 @@ def finish_bundle_outputs(self, results):
 
       if isinstance(result, WindowedValue):
         windowed_value = result
-      elif isinstance(result, TimestampedValue):
-        value = result.value
-        timestamp = result.timestamp
-        assign_context = NoContext(value, timestamp)
-        windowed_value = WindowedValue(
-            value, timestamp, self.window_fn.assign(assign_context))
       else:
-        value = result
-        timestamp = -1
-        assign_context = NoContext(value)
-        windowed_value = WindowedValue(
-            value, timestamp, self.window_fn.assign(assign_context))
+        raise RuntimeError('Finish Bundle should only output WindowedValue ' +\
+                           'type but got %s' % type(result))
 
       if tag is None:
         self.main_receivers.receive(windowed_value)
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index e7126610935b..5948460579de 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -30,8 +30,10 @@
 from apache_beam.metrics import Metrics
 from apache_beam.metrics.metric import MetricsFilter
 from apache_beam.io.iobase import Read
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.options.pipeline_options import TypeOptions
 import apache_beam.pvalue as pvalue
+from apache_beam.test_pipeline import TestPipeline
+from apache_beam.transforms import window
 import apache_beam.transforms.combiners as combine
 from apache_beam.transforms.display import DisplayData, DisplayDataItem
 from apache_beam.transforms.ptransform import PTransform
@@ -40,7 +42,7 @@
 from apache_beam.typehints import with_input_types
 from apache_beam.typehints import with_output_types
 from apache_beam.typehints.typehints_test import TypeHintTestCase
-from apache_beam.options.pipeline_options import TypeOptions
+from apache_beam.utils.windowed_value import WindowedValue
 
 
 # Disable frequent lint warning due to pipe operator for chaining transforms.
@@ -280,7 +282,7 @@ def process(self, element):
         pass
 
       def finish_bundle(self):
-        yield 'finish'
+        yield WindowedValue('finish', -1, [window.GlobalWindow()])
 
     pipeline = TestPipeline()
     pcoll = pipeline | 'Start' >> beam.Create([1, 2, 3])

From 672a2c400badd8cd33edb9585d2325b9f265c963 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Mon, 8 May 2017 15:26:50 -0700
Subject: [PATCH 142/387] Remove sdk & example README.md that are covered in
 web site

---
 .gitignore                                    |   1 +
 examples/java/README.md                       |  64 +---
 .../beam/examples/complete/game/README.md     | 131 --------
 pom.xml                                       |   2 +
 sdks/python/README.md                         | 298 ------------------
 .../examples/complete/game/README.md          |  69 ----
 6 files changed, 5 insertions(+), 560 deletions(-)
 delete mode 100644 examples/java8/src/main/java/org/apache/beam/examples/complete/game/README.md
 delete mode 100644 sdks/python/README.md
 delete mode 100644 sdks/python/apache_beam/examples/complete/game/README.md

diff --git a/.gitignore b/.gitignore
index 9cfae09cb4a1..1ecb993777b0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ sdks/python/**/*.so
 sdks/python/**/*.egg
 sdks/python/LICENSE
 sdks/python/NOTICE
+sdks/python/README.md
 
 # Ignore IntelliJ files.
 .idea/
diff --git a/examples/java/README.md b/examples/java/README.md
index d891fb87842d..75b70dde8528 100644
--- a/examples/java/README.md
+++ b/examples/java/README.md
@@ -40,69 +40,9 @@ demonstrates some best practices for instrumenting your pipeline code.
 
 1. [`WindowedWordCount`](https://github.com/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java) shows how to run the same pipeline over either unbounded PCollections in streaming mode or bounded PCollections in batch mode.
 
-## Building and Running
+## Running Examples
 
-Change directory into `examples/java` and run the examples:
-
-    mvn compile exec:java \
-    -Dexec.mainClass=<MAIN CLASS> \
-    -Dexec.args="<EXAMPLE-SPECIFIC ARGUMENTS>"
-
-Alternatively, you may choose to bundle all dependencies into a single JAR and
-execute it outside of the Maven environment.
-
-### Direct Runner
-
-You can execute the `WordCount` pipeline on your local machine as follows:
-
-    mvn compile exec:java \
-    -Dexec.mainClass=org.apache.beam.examples.WordCount \
-    -Dexec.args="--inputFile=<LOCAL INPUT FILE> --output=<LOCAL OUTPUT FILE>"
-
-To create the bundled JAR of the examples and execute it locally:
-
-    mvn package
-
-    java -cp examples/java/target/beam-examples-java-bundled-<VERSION>.jar \
-    org.apache.beam.examples.WordCount \
-    --inputFile=<INPUT FILE PATTERN> --output=<OUTPUT FILE>
-
-### Google Cloud Dataflow Runner
-
-After you have followed the general Cloud Dataflow
-[prerequisites and setup](https://beam.apache.org/documentation/runners/dataflow/), you can execute
-the pipeline on fully managed resources in Google Cloud Platform:
-
-    mvn compile exec:java \
-    -Dexec.mainClass=org.apache.beam.examples.WordCount \
-    -Dexec.args="--project=<YOUR CLOUD PLATFORM PROJECT ID> \
-    --tempLocation=<YOUR CLOUD STORAGE LOCATION> \
-    --runner=DataflowRunner"
-
-Make sure to use your project id, not the project number or the descriptive name.
-The Google Cloud Storage location should be entered in the form of
-`gs://bucket/path/to/staging/directory`.
-
-To create the bundled JAR of the examples and execute it in Google Cloud Platform:
-
-    mvn package
-
-    java -cp examples/java/target/beam-examples-java-bundled-<VERSION>.jar \
-    org.apache.beam.examples.WordCount \
-    --project=<YOUR CLOUD PLATFORM PROJECT ID> \
-    --tempLocation=<YOUR CLOUD STORAGE LOCATION> \
-    --runner=DataflowRunner
-
-## Other Examples
-
-Other examples can be run similarly by replacing the `WordCount` class path with the example classpath, e.g.
-`org.apache.beam.examples.cookbook.CombinePerKeyExamples`,
-and adjusting runtime options under the `Dexec.args` parameter, as specified in
-the example itself.
-
-Note that when running Maven on Microsoft Windows platform, backslashes (`\`)
-under the `Dexec.args` parameter should be escaped with another backslash. For
-example, input file pattern of `c:\*.txt` should be entered as `c:\\*.txt`.
+See [Apache Beam WordCount Example](https://beam.apache.org/get-started/wordcount-example/) for information on running these examples.
 
 ## Beyond Word Count
 
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/README.md b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/README.md
deleted file mode 100644
index fdce05cd2405..000000000000
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/README.md
+++ /dev/null
@@ -1,131 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-# 'Gaming' examples
-
-
-This directory holds a series of example Apache Beam pipelines in a simple 'mobile
-gaming' domain. They all require Java 8.  Each pipeline successively introduces
-new concepts, and gives some examples of using Java 8 syntax in constructing
-Beam pipelines. Other than usage of Java 8 lambda expressions, the concepts
-that are used apply equally well in Java 7.
-
-In the gaming scenario, many users play, as members of different teams, over
-the course of a day, and their actions are logged for processing. Some of the
-logged game events may be late-arriving, if users play on mobile devices and go
-transiently offline for a period.
-
-The scenario includes not only "regular" users, but "robot users", which have a
-higher click rate than the regular users, and may move from team to team.
-
-The first two pipelines in the series use pre-generated batch data samples. The
-second two pipelines read from a [PubSub](https://cloud.google.com/pubsub/)
-topic input.  For these examples, you will also need to run the
-`injector.Injector` program, which generates and publishes the gaming data to
-PubSub. The javadocs for each pipeline have more detailed information on how to
-run that pipeline.
-
-All of these pipelines write their results to BigQuery table(s).
-
-
-## The pipelines in the 'gaming' series
-
-### UserScore
-
-The first pipeline in the series is `UserScore`. This pipeline does batch
-processing of data collected from gaming events. It calculates the sum of
-scores per user, over an entire batch of gaming data (collected, say, for each
-day). The batch processing will not include any late data that arrives after
-the day's cutoff point.
-
-### HourlyTeamScore
-
-The next pipeline in the series is `HourlyTeamScore`. This pipeline also
-processes data collected from gaming events in batch. It builds on `UserScore`,
-but uses [fixed windows](https://beam.apache.org/documentation/programming-guide/#windowing), by
-default an hour in duration. It calculates the sum of scores per team, for each
-window, optionally allowing specification of two timestamps before and after
-which data is filtered out. This allows a model where late data collected after
-the intended analysis window can be included in the analysis, and any late-
-arriving data prior to the beginning of the analysis window can be removed as
-well.
-
-By using windowing and adding element timestamps, we can do finer-grained
-analysis than with the `UserScore` pipeline — we're now tracking scores for
-each hour rather than over the course of a whole day. However, our batch
-processing is high-latency, in that we don't get results from plays at the
-beginning of the batch's time period until the complete batch is processed.
-
-### LeaderBoard
-
-The third pipeline in the series is `LeaderBoard`. This pipeline processes an
-unbounded stream of 'game events' from a PubSub topic. The calculation of the
-team scores uses fixed windowing based on event time (the time of the game play
-event), not processing time (the time that an event is processed by the
-pipeline). The pipeline calculates the sum of scores per team, for each window.
-By default, the team scores are calculated using one-hour windows.
-
-In contrast — to demo another windowing option — the user scores are calculated
-using a global window, which periodically (every ten minutes) emits cumulative
-user score sums.
-
-In contrast to the previous pipelines in the series, which used static, finite
-input data, here we're using an unbounded data source, which lets us provide
-_speculative_ results, and allows handling of late data, at much lower latency.
-E.g., we could use the early/speculative results to keep a 'leaderboard'
-updated in near-realtime. Our handling of late data lets us generate correct
-results, e.g. for 'team prizes'. We're now outputing window results as they're
-calculated, giving us much lower latency than with the previous batch examples.
-
-### GameStats
-
-The fourth pipeline in the series is `GameStats`. This pipeline builds
-on the `LeaderBoard` functionality — supporting output of speculative and late
-data — and adds some "business intelligence" analysis: identifying abuse
-detection. The pipeline derives the Mean user score sum for a window, and uses
-that information to identify likely spammers/robots. (The injector is designed
-so that the "robots" have a higher click rate than the "real" users). The robot
-users are then filtered out when calculating the team scores.
-
-Additionally, user sessions are tracked: that is, we find bursts of user
-activity using session windows. Then, the mean session duration information is
-recorded in the context of subsequent fixed windowing. (This could be used to
-tell us what games are giving us greater user retention).
-
-### Running the PubSub Injector
-
-The `LeaderBoard` and `GameStats` example pipelines read unbounded data
-from a PubSub topic.
-
-Use the `injector.Injector` program to generate this data and publish to a
-PubSub topic. See the `Injector`javadocs for more information on how to run the
-injector. Set up the injector before you start one of these pipelines. Then,
-when you start the pipeline, pass as an argument the name of that PubSub topic.
-See the pipeline javadocs for the details.
-
-## Viewing the results in BigQuery
-
-All of the pipelines write their results to BigQuery.  `UserScore` and
-`HourlyTeamScore` each write one table, and `LeaderBoard` and
-`GameStats` each write two. The pipelines have default table names that
-you can override when you start up the pipeline if those tables already exist.
-
-Depending on the windowing intervals defined in a given pipeline, you may have
-to wait for a while (more than an hour) before you start to see results written
-to the BigQuery tables.
diff --git a/pom.xml b/pom.xml
index e3fa7d825381..9ca876ebf69c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1543,6 +1543,7 @@
                   <include>**/*.egg-info/</include>
                   <include>**/sdks/python/LICENSE</include>
                   <include>**/sdks/python/NOTICE</include>
+                  <include>**/sdks/python/README.md</include>
                 </includes>
                 <followSymlinks>false</followSymlinks>
               </fileset>
@@ -1680,6 +1681,7 @@
                   <includes>
                     <include>LICENSE</include>
                     <include>NOTICE</include>
+                    <include>README.md</include>
                   </includes>
                 </resource>
               </resources>              
diff --git a/sdks/python/README.md b/sdks/python/README.md
deleted file mode 100644
index 7836a04ead89..000000000000
--- a/sdks/python/README.md
+++ /dev/null
@@ -1,298 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-# Apache Beam - Python SDK
-
-[Apache Beam](http://beam.apache.org) is a unified model for defining both batch and streaming data-parallel processing pipelines. Beam provides a set of language-specific SDKs for constructing pipelines. These pipelines can be executed on distributed processing backends like [Apache Spark](http://spark.apache.org/), [Apache Flink](http://flink.apache.org), and [Google Cloud Dataflow](http://cloud.google.com/dataflow).
-
-Apache Beam for Python provides access to Beam capabilities from the Python programming language.
-
-## Table of Contents
-  * [Overview of the Beam Programming Model](#overview-of-the-programming-model)
-  * [Getting Started](#getting-started)
-  * [A Quick Tour of the Source Code](#a-quick-tour-of-the-source-code)
-  * [Simple Examples](#simple-examples)
-      * [Basic pipeline](#basic-pipeline)
-      * [Basic pipeline (with Map)](#basic-pipeline-with-map)
-      * [Basic pipeline (with FlatMap)](#basic-pipeline-with-flatmap)
-      * [Basic pipeline (with FlatMap and yield)](#basic-pipeline-with-flatmap-and-yield)
-      * [Counting words](#counting-words)
-      * [Counting words with GroupByKey](#counting-words-with-groupbykey)
-      * [Type hints](#type-hints)
-      * [BigQuery](#bigquery)
-      * [Combiner examples](#combiner-examples)
-  * [Organizing Your Code](#organizing-your-code)
-  * [Contact Us](#contact-us)
-
-## Overview of the Programming Model
-
-The key concepts of the programming model are:
-
-* PCollection - represents a collection of data, which could be bounded or unbounded in size.
-* PTransform - represents a computation that transforms input PCollections into output
-PCollections.
-* Pipeline - manages a directed acyclic graph of PTransforms and PCollections that is ready
-for execution.
-* Runner - specifies where and how the Pipeline should execute.
-
-For a further, detailed introduction, please read the
-[Beam Programming Model](http://beam.apache.org/documentation/programming-guide).
-
-## Getting Started
-
-See [Apache Beam Python SDK Quickstart](https://beam.apache.org/get-started/quickstart-py/).
-
-## A Quick Tour of the Source Code
-
-With your virtual environment active, you can follow along this tour by running a `pydoc` server on a local port of your choosing (this example uses port 8888):
-
-```
-pydoc -p 8888
-```
-
-Open your browser and go to
-http://localhost:8888/apache_beam.html
-
-Some interesting classes to navigate to:
-
-* `PCollection`, in file
-[`apache_beam/pvalue.py`](http://localhost:8888/apache_beam.pvalue.html)
-* `PTransform`, in file
-[`apache_beam/transforms/ptransform.py`](http://localhost:8888/apache_beamtransforms.ptransform.html)
-* `FlatMap`, `GroupByKey`, and `Map`, in file
-[`apache_beam/transforms/core.py`](http://localhost:8888/apache_beam.transforms.core.html)
-* combiners, in file
-[`apache_beam/transforms/combiners.py`](http://localhost:8888/apache_beam.transforms.combiners.html)
-
-Make sure you installed the package first. If not, run `python setup.py install`, then run pydoc with `pydoc -p 8888`.
-
-## Simple Examples
-
-The following examples demonstrate some basic, fundamental concepts for using Apache Beam's Python SDK. For more detailed examples, Beam provides a [directory of examples](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples) for Python.
-
-### Basic pipeline
-
-A basic pipeline will take as input an iterable, apply the
-beam.Create `PTransform`, and produce a `PCollection` that can
-be written to a file or modified by further `PTransform`s.
-The `>>` operator is used to label `PTransform`s and
-the `|` operator is used to chain them.
-
-```python
-# Standard imports
-import apache_beam as beam
-# Create a pipeline executing on a direct runner (local, non-cloud).
-p = beam.Pipeline('DirectRunner')
-# Create a PCollection with names and write it to a file.
-(p
- | 'add names' >> beam.Create(['Ann', 'Joe'])
- | 'save' >> beam.io.WriteToText('./names'))
-# Execute the pipeline.
-p.run()
-```
-
-### Basic pipeline (with Map)
-
-The `Map` `PTransform` returns one output per input. It takes a callable that is applied to each element of the input `PCollection` and returns an element to the output `PCollection`.
-
-```python
-import apache_beam as beam
-p = beam.Pipeline('DirectRunner')
-# Read a file containing names, add a greeting to each name, and write to a file.
-(p
- | 'load names' >> beam.io.ReadFromText('./names')
- | 'add greeting' >> beam.Map(lambda name, msg: '%s, %s!' % (msg, name), 'Hello')
- | 'save' >> beam.io.WriteToText('./greetings'))
-p.run()
-```
-
-### Basic pipeline (with FlatMap)
-
-A `FlatMap` is like a `Map` except its callable returns a (possibly
-empty) iterable of elements for the output `PCollection`.
-
-The `FlatMap` transform returns zero to many output per input. It accepts a callable that is applied to each element of the input `PCollection` and returns an iterable with zero or more elements to the output `PCollection`.
-
-```python
-import apache_beam as beam
-p = beam.Pipeline('DirectRunner')
-# Read a file containing names, add two greetings to each name, and write to a file.
-(p
- | 'load names' >> beam.io.ReadFromText('./names')
- | 'add greetings' >> beam.FlatMap(
-    lambda name, messages: ['%s %s!' % (msg, name) for msg in messages],
-    ['Hello', 'Hola'])
- | 'save' >> beam.io.WriteToText('./greetings'))
-p.run()
-```
-
-### Basic pipeline (with FlatMap and yield)
-
-The callable of a `FlatMap` can be a generator, that is,
-a function using `yield`.
-
-```python
-import apache_beam as beam
-p = beam.Pipeline('DirectRunner')
-# Read a file containing names, add two greetings to each name
-# (with FlatMap using a yield generator), and write to a file.
-def add_greetings(name, messages):
-  for msg in messages:
-    yield '%s %s!' % (msg, name)
-
-(p
- | 'load names' >> beam.io.ReadFromText('./names')
- | 'add greetings' >> beam.FlatMap(add_greetings, ['Hello', 'Hola'])
- | 'save' >> beam.io.WriteToText('./greetings'))
-p.run()
-```
-
-### Counting words
-
-This example shows how to read a text file from [Google Cloud Storage](https://cloud.google.com/storage/) and count its words.
-
-```python
-import re
-import apache_beam as beam
-p = beam.Pipeline('DirectRunner')
-(p
- | 'read' >> beam.io.ReadFromText('gs://dataflow-samples/shakespeare/kinglear.txt')
- | 'split' >> beam.FlatMap(lambda x: re.findall(r'\w+', x))
- | 'count words' >> beam.combiners.Count.PerElement()
- | 'save' >> beam.io.WriteToText('./word_count'))
-p.run()
-```
-
-### Counting words with GroupByKey
-
-This is a somewhat forced example of `GroupByKey` to count words as the previous example did, but without using `beam.combiners.Count.PerElement`. As shown in the example, you can use a wildcard to specify the text file source.
-
-```python
-import re
-import apache_beam as beam
-p = beam.Pipeline('DirectRunner')
-class MyCountTransform(beam.PTransform):
-  def expand(self, pcoll):
-    return (pcoll
-            | 'one word' >> beam.Map(lambda word: (word, 1))
-            # GroupByKey accepts a PCollection of (word, 1) elements and
-            # outputs a PCollection of (word, [1, 1, ...])
-            | 'group words' >> beam.GroupByKey()
-            | 'count words' >> beam.Map(lambda (word, counts): (word, len(counts))))
-
-(p
- | 'read' >> beam.io.ReadFromText('./names*')
- | 'split' >> beam.FlatMap(lambda x: re.findall(r'\w+', x))
- | MyCountTransform()
- | 'write' >> beam.io.WriteToText('./word_count'))
-p.run()
-```
-
-### Type hints
-
-In some cases, providing type hints can improve the efficiency
-of the data encoding.
-
-```python
-import apache_beam as beam
-from apache_beam.typehints import typehints
-p = beam.Pipeline('DirectRunner')
-(p
- | 'read' >> beam.io.ReadFromText('./names')
- | 'add types' >> beam.Map(lambda x: (x, 1)).with_output_types(typehints.KV[str, int])
- | 'group words' >> beam.GroupByKey()
- | 'save' >> beam.io.WriteToText('./typed_names'))
-p.run()
-```
-
-### BigQuery
-
-This example reads weather data from a BigQuery table, calculates the number of tornadoes per month, and writes the results to a table you specify.
-
-```python
-import apache_beam as beam
-project = 'DESTINATION-PROJECT-ID'
-input_table = 'clouddataflow-readonly:samples.weather_stations'
-output_table = 'DESTINATION-DATASET.DESTINATION-TABLE'
-
-p = beam.Pipeline(argv=['--project', project])
-(p
- | 'read' >> beam.Read(beam.io.BigQuerySource(input_table))
- | 'months with tornadoes' >> beam.FlatMap(
-    lambda row: [(int(row['month']), 1)] if row['tornado'] else [])
- | 'monthly count' >> beam.CombinePerKey(sum)
- | 'format' >> beam.Map(lambda (k, v): {'month': k, 'tornado_count': v})
- | 'save' >> beam.Write(
-    beam.io.BigQuerySink(
-        output_table,
-        schema='month:INTEGER, tornado_count:INTEGER',
-        create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
-        write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)))
-p.run()
-```
-
-This pipeline, like the one above, calculates the number of tornadoes per month, but it uses a query to filter out the input instead of using the whole table.
-
-```python
-import apache_beam as beam
-project = 'DESTINATION-PROJECT-ID'
-output_table = 'DESTINATION-DATASET.DESTINATION-TABLE'
-input_query = 'SELECT month, COUNT(month) AS tornado_count ' \
-        'FROM [clouddataflow-readonly:samples.weather_stations] ' \
-        'WHERE tornado=true GROUP BY month'
-p = beam.Pipeline(argv=['--project', project])
-(p
- | 'read' >> beam.Read(beam.io.BigQuerySource(query=input_query))
- | 'save' >> beam.Write(beam.io.BigQuerySink(
-    output_table,
-    schema='month:INTEGER, tornado_count:INTEGER',
-    create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
-    write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)))
-p.run()
-```
-
-### Combiner Examples
-
-Combiner transforms use "reducing" functions, such as sum, min, or max, to combine multiple values of a `PCollection` into a single value.
-
-```python
-import apache_beam as beam
-p = beam.Pipeline('DirectRunner')
-
-SAMPLE_DATA = [('a', 1), ('b', 10), ('a', 2), ('a', 3), ('b', 20)]
-
-(p
- | beam.Create(SAMPLE_DATA)
- | beam.CombinePerKey(sum)
- | beam.io.WriteToText('./sums'))
-p.run()
-```
-
-The [combiners_test.py](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/examples/cookbook/combiners_test.py) file contains more combiner examples.
-
-## Organizing Your Code
-
-Many projects will grow to multiple source code files. It is recommended that you organize your project so that all code involved in running your pipeline can be built as a Python package. This way, the package can easily be installed in the VM workers executing the job.
-
-Follow the [Juliaset example](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/complete/juliaset). If the code is organized in this fashion, you can use the `--setup_file` command line option to create a source distribution out of the project files, stage the resulting tarball, and later install it in the workers executing the job.
-
-## More Information
-
-Please report any issues on [JIRA](https://issues.apache.org/jira/browse/BEAM/component/12328910).
-
-If you’re interested in contributing to the Beam SDK, start by reading the [Contribute](http://beam.apache.org/contribute/) guide.
diff --git a/sdks/python/apache_beam/examples/complete/game/README.md b/sdks/python/apache_beam/examples/complete/game/README.md
deleted file mode 100644
index 39677e477af6..000000000000
--- a/sdks/python/apache_beam/examples/complete/game/README.md
+++ /dev/null
@@ -1,69 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-# 'Gaming' examples
-
-This directory holds a series of example Dataflow pipelines in a simple 'mobile
-gaming' domain. Each pipeline successively introduces new concepts.
-
-In the gaming scenario, many users play, as members of different teams, over
-the course of a day, and their actions are logged for processing. Some of the
-logged game events may be late-arriving, if users play on mobile devices and go
-transiently offline for a period.
-
-The scenario includes not only "regular" users, but "robot users", which have a
-higher click rate than the regular users, and may move from team to team.
-
-The first two pipelines in the series use pre-generated batch data samples.
-
-All of these pipelines write their results to Google BigQuery table(s).
-
-## The pipelines in the 'gaming' series
-
-### user_score
-
-The first pipeline in the series is `user_score`. This pipeline does batch
-processing of data collected from gaming events. It calculates the sum of
-scores per user, over an entire batch of gaming data (collected, say, for each
-day). The batch processing will not include any late data that arrives after
-the day's cutoff point.
-
-### hourly_team_score
-
-The next pipeline in the series is `hourly_team_score`. This pipeline also
-processes data collected from gaming events in batch. It builds on `user_score`,
-but uses [fixed windows](https://beam.apache.org/documentation/programming-guide/#windowing),
-by default an hour in duration. It calculates the sum of scores per team, for
-each window, optionally allowing specification of two timestamps before and
-after which data is filtered out. This allows a model where late data collected
-after the intended analysis window can be included in the analysis, and any
-late-arriving data prior to the beginning of the analysis window can be removed
-as well.
-
-By using windowing and adding element timestamps, we can do finer-grained
-analysis than with the `UserScore` pipeline — we're now tracking scores for
-each hour rather than over the course of a whole day. However, our batch
-processing is high-latency, in that we don't get results from plays at the
-beginning of the batch's time period until the complete batch is processed.
-
-## Viewing the results in BigQuery
-
-All of the pipelines write their results to BigQuery. `user_score` and
-`hourly_team_score` each write one table. The pipelines have default table names
-that you can override when you start up the pipeline if those tables already
-exist.

From 7d165a73070468ff3a8907bcd5c3a3c4972d79e5 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Mon, 8 May 2017 16:01:40 -0700
Subject: [PATCH 143/387] [BEAM-2211] Move PathValidator into GCP-Core

For now, this does not need to be a Beam concept
---
 .../java/org/apache/beam/runners/dataflow/DataflowRunner.java | 2 +-
 .../beam/runners/dataflow/DataflowPipelineTranslatorTest.java | 2 +-
 .../apache/beam/sdk/extensions/gcp/options/GcpOptions.java    | 2 +-
 .../apache/beam/sdk/extensions/gcp/options/GcsOptions.java    | 4 ++--
 .../{util => extensions/gcp/storage}/GcsPathValidator.java    | 4 ++--
 .../beam/sdk/extensions/gcp/storage}/NoopPathValidator.java   | 3 ++-
 .../beam/sdk/extensions/gcp/storage}/PathValidator.java       | 3 ++-
 .../main/java/org/apache/beam/sdk/util/NoopPathValidator.java | 4 ++--
 .../gcp/storage}/GcsPathValidatorTest.java                    | 3 ++-
 9 files changed, 15 insertions(+), 12 deletions(-)
 rename sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/{util => extensions/gcp/storage}/GcsPathValidator.java (97%)
 rename sdks/java/{core/src/main/java/org/apache/beam/sdk/io/fs => extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage}/NoopPathValidator.java (94%)
 rename sdks/java/{core/src/main/java/org/apache/beam/sdk/io/fs => extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage}/PathValidator.java (94%)
 rename sdks/java/{core => extensions/google-cloud-platform-core}/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java (90%)
 rename sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/{util => extensions/gcp/storage}/GcsPathValidatorTest.java (97%)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 250c0641e426..2ef87374956d 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -83,11 +83,11 @@
 import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.UnboundedSource;
-import org.apache.beam.sdk.io.fs.PathValidator;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 9040f8f01c4f..93c1e5b109cc 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -77,6 +77,7 @@
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
+import org.apache.beam.sdk.extensions.gcp.storage.GcsPathValidator;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -93,7 +94,6 @@
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.util.GcsPathValidator;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.KV;
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index 985520f03a00..1e25560974b0 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -48,7 +48,7 @@
 import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory;
 import org.apache.beam.sdk.extensions.gcp.auth.GcpCredentialFactory;
 import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer;
-import org.apache.beam.sdk.io.fs.PathValidator;
+import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java
index 78e233e967c1..7ac9b692638e 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.java
@@ -26,14 +26,14 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
-import org.apache.beam.sdk.io.fs.PathValidator;
+import org.apache.beam.sdk.extensions.gcp.storage.GcsPathValidator;
+import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
 import org.apache.beam.sdk.options.ApplicationNameOptions;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.Hidden;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.GcsPathValidator;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.InstanceBuilder;
 
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java
similarity index 97%
rename from sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
rename to sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java
index c4e557bd41de..e7257b24b8cc 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java
@@ -15,15 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.extensions.gcp.storage;
 
 import static com.google.common.base.Preconditions.checkArgument;
 
 import java.io.IOException;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
-import org.apache.beam.sdk.io.fs.PathValidator;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/NoopPathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/NoopPathValidator.java
similarity index 94%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/NoopPathValidator.java
rename to sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/NoopPathValidator.java
index d5be8f0fc5fb..79b87324d573 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/NoopPathValidator.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/NoopPathValidator.java
@@ -15,9 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.io.fs;
+package org.apache.beam.sdk.extensions.gcp.storage;
 
 import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/PathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/PathValidator.java
similarity index 94%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/PathValidator.java
rename to sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/PathValidator.java
index b88a33e75727..cc769c870313 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/PathValidator.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/PathValidator.java
@@ -15,9 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.io.fs;
+package org.apache.beam.sdk.extensions.gcp.storage;
 
 import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.fs.ResourceId;
 
 /**
  * <b>For internal use only; no backwards compatibility guarantees.</b>
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
similarity index 90%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
rename to sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
index 0015e5945767..85b68b21cfdf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
@@ -17,12 +17,12 @@
  */
 package org.apache.beam.sdk.util;
 
-import org.apache.beam.sdk.io.fs.PathValidator;
+import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
 
 /**
- * @deprecated use {@link org.apache.beam.sdk.io.fs.NoopPathValidator}.
+ * @deprecated use {@link org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator}.
  */
 @Deprecated
 public class NoopPathValidator implements PathValidator {
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidatorTest.java
similarity index 97%
rename from sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java
rename to sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidatorTest.java
index 65fb22891a34..91ac46c9c996 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidatorTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.util;
+package org.apache.beam.sdk.extensions.gcp.storage;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.when;
@@ -23,6 +23,7 @@
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.junit.Before;
 import org.junit.Rule;

From f03c04a787343c3710355c84a105582cdc815469 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 09:46:38 -0700
Subject: [PATCH 144/387] Handle Errors in the DirectRunner

When a worker dies because of an error, propagate that error and fail
the Pipeline.
---
 .../runners/direct/CompletionCallback.java    |  6 +++++
 .../ExecutorServiceParallelExecutor.java      | 27 ++++++++++++++-----
 .../runners/direct/TransformExecutor.java     | 12 +++++++--
 .../apache/beam/runners/direct/MockClock.java |  2 +-
 .../runners/direct/TransformExecutorTest.java |  5 ++++
 5 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
index 0af22c83707e..417fa0965875 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
@@ -40,4 +40,10 @@ CommittedResult handleResult(
    * Handle a result that terminated abnormally due to the provided {@link Exception}.
    */
   void handleException(CommittedBundle<?> inputBundle, Exception t);
+
+  /**
+   * Handle a result that terminated abnormally due to the provided {@link Error}. The pipeline
+   * should be shut down, and the Error propagated.
+  */
+  void handleError(Error err);
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index b7f473241e9a..71ab4cc0e08c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -285,8 +285,15 @@ public State waitUntilFinish(Duration duration) throws Exception {
         // there are no updates to process and no updates will ever be published because the
         // executor is shutdown
         return pipelineState.get();
-      } else if (update != null && update.exception.isPresent()) {
-        throw update.exception.get();
+      } else if (update != null && update.thrown.isPresent()) {
+        Throwable thrown = update.thrown.get();
+        if (thrown instanceof Exception) {
+          throw (Exception) thrown;
+        } else if (thrown instanceof Error) {
+          throw (Error) thrown;
+        } else {
+          throw new Exception("Unknown Type of Throwable", thrown);
+        }
       }
     }
     return pipelineState.get();
@@ -380,6 +387,11 @@ public final void handleException(CommittedBundle<?> inputBundle, Exception e) {
       allUpdates.offer(ExecutorUpdate.fromException(e));
       outstandingWork.decrementAndGet();
     }
+
+    @Override
+    public void handleError(Error err) {
+      visibleUpdates.add(VisibleExecutorUpdate.fromError(err));
+    }
   }
 
   /**
@@ -424,7 +436,7 @@ public static ExecutorUpdate fromException(Exception e) {
    * return normally or throw an exception.
    */
   private static class VisibleExecutorUpdate {
-    private final Optional<? extends Exception> exception;
+    private final Optional<? extends Throwable> thrown;
     @Nullable
     private final State newState;
 
@@ -432,6 +444,10 @@ public static VisibleExecutorUpdate fromException(Exception e) {
       return new VisibleExecutorUpdate(null, e);
     }
 
+    public static VisibleExecutorUpdate fromError(Error err) {
+      return new VisibleExecutorUpdate(State.FAILED, err);
+    }
+
     public static VisibleExecutorUpdate finished() {
       return new VisibleExecutorUpdate(State.DONE, null);
     }
@@ -440,15 +456,14 @@ public static VisibleExecutorUpdate cancelled() {
       return new VisibleExecutorUpdate(State.CANCELLED, null);
     }
 
-    private VisibleExecutorUpdate(State newState, @Nullable Exception exception) {
-      this.exception = Optional.fromNullable(exception);
+    private VisibleExecutorUpdate(State newState, @Nullable Throwable exception) {
+      this.thrown = Optional.fromNullable(exception);
       this.newState = newState;
     }
 
     public State getNewState() {
       return newState;
     }
-
   }
 
   private class MonitorRunnable implements Runnable {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
index 8e1515bb1901..56f8650a1034 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
@@ -26,6 +26,8 @@
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.util.WindowedValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A {@link Callable} responsible for constructing a {@link TransformEvaluator} from a
@@ -36,6 +38,8 @@
  * that it is being executed on.
  */
 class TransformExecutor<T> implements Runnable {
+  private static final Logger LOG = LoggerFactory.getLogger(TransformExecutor.class);
+
   public static <T> TransformExecutor<T> create(
       EvaluationContext context,
       TransformEvaluatorFactory factory,
@@ -112,6 +116,10 @@ public void run() {
         throw (RuntimeException) e;
       }
       throw new RuntimeException(e);
+    } catch (Error err) {
+      LOG.error("Error occurred within {}", this, err);
+      onComplete.handleError(err);
+      throw err;
     } finally {
       // Report the physical metrics from the end of this step.
       context.getMetrics().commitPhysical(inputBundle, metricsContainer.getCumulative());
@@ -162,8 +170,8 @@ private TransformResult<T> finishBundle(
       TransformEvaluator<T> evaluator, MetricsContainer metricsContainer,
       Collection<ModelEnforcement<T>> enforcements)
       throws Exception {
-    TransformResult<T> result = evaluator.finishBundle()
-        .withLogicalMetricUpdates(metricsContainer.getCumulative());
+    TransformResult<T> result =
+        evaluator.finishBundle().withLogicalMetricUpdates(metricsContainer.getCumulative());
     CommittedResult outputs = onComplete.handleResult(inputBundle, result);
     for (ModelEnforcement<T> enforcement : enforcements) {
       enforcement.afterFinish(inputBundle, result, outputs.getOutputs());
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java
index 11ecbff91d52..9275e3c1096b 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java
@@ -28,7 +28,7 @@
  *
  * <p>For uses of the {@link Clock} interface in unit tests.
  */
-public class MockClock implements Clock {
+class MockClock implements Clock {
 
   private Instant now;
 
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
index dc0ef7ca7828..86412a0234e6 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
@@ -434,6 +434,11 @@ public void handleException(CommittedBundle<?> inputBundle, Exception e) {
       handledException = e;
       onMethod.countDown();
     }
+
+    @Override
+    public void handleError(Error err) {
+      throw err;
+    }
   }
 
   private static class TestEnforcementFactory implements ModelEnforcementFactory {

From b0a326148e537f4c8a5de59f6a5ffdc1ccabea05 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 09:47:47 -0700
Subject: [PATCH 145/387] Shade Dependencies of the DirectRunner

Add an API Surface Test
---
 runners/direct-java/pom.xml                   | 79 +++++++++++++++++--
 .../GroupAlsoByWindowEvaluatorFactory.java    |  6 +-
 .../direct/DirectRunnerApiSurfaceTest.java    | 55 +++++++++++++
 3 files changed, 133 insertions(+), 7 deletions(-)
 create mode 100644 runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java

diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index ba3cd3ea04b1..19ee81a02c08 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -77,6 +77,76 @@
         </executions>
       </plugin>
 
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>bundle-and-repackage</id>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <artifactSet>
+                <includes>
+                  <include>com.google.guava:guava</include>
+                  <include>com.google.protobuf:protobuf-java</include>
+                  <include>org.apache.beam:beam-runners-core-construction-java</include>
+                  <include>org.apache.beam:beam-runners-core-java</include>
+                  <include>org.apache.beam:beam-sdks-common-runner-api</include>
+                </includes>
+              </artifactSet>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <relocations>
+                <relocation>
+                  <pattern>org.apache.beam.runners.core</pattern>
+                  <shadedPattern>
+                    org.apache.beam.runners.direct.repackaged.runners.core
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.beam.sdk.common</pattern>
+                  <shadedPattern>
+                    org.apache.beam.runners.direct.repackaged.sdk.common
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.google.common</pattern>
+                  <shadedPattern>
+                    org.apache.beam.runners.direct.repackaged.com.google.common
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.google.protobuf</pattern>
+                  <shadedPattern>
+                    org.apache.beam.runners.direct.repackaged.com.google.protobuf
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.google.thirdparty</pattern>
+                  <shadedPattern>
+                    org.apache.beam.runners.direct.repackaged.com.google.thirdparty
+                  </shadedPattern>
+                </relocation>
+              </relocations>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
       <!-- Coverage analysis for unit tests. -->
       <plugin>
         <groupId>org.jacoco</groupId>
@@ -93,18 +163,17 @@
 
     <dependency>
       <groupId>org.apache.beam</groupId>
-      <artifactId>beam-runners-core-construction-java</artifactId>
+      <artifactId>beam-sdks-common-runner-api</artifactId>
     </dependency>
 
     <dependency>
       <groupId>org.apache.beam</groupId>
-      <artifactId>beam-runners-core-java</artifactId>
+      <artifactId>beam-runners-core-construction-java</artifactId>
     </dependency>
 
     <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client-protobuf</artifactId>
-      <scope>runtime</scope>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-runners-core-java</artifactId>
     </dependency>
 
     <dependency>
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 41c797f840b2..1d079d9672d5 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -40,6 +40,7 @@
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -161,13 +162,14 @@ public void processElement(WindowedValue<KeyedWorkItem<K, V>> element) throws Ex
       CopyOnAccessInMemoryStateInternals stateInternals =
           (CopyOnAccessInMemoryStateInternals) stepContext.stateInternals();
       DirectTimerInternals timerInternals = stepContext.timerInternals();
+      RunnerApi.Trigger runnerApiTrigger =
+          Triggers.toProto(windowingStrategy.getTrigger());
       ReduceFnRunner<K, V, Iterable<V>, BoundedWindow> reduceFnRunner =
           new ReduceFnRunner<>(
               key,
               windowingStrategy,
               ExecutableTriggerStateMachine.create(
-                  TriggerStateMachines.stateMachineForTrigger(
-                      Triggers.toProto(windowingStrategy.getTrigger()))),
+                  TriggerStateMachines.stateMachineForTrigger(runnerApiTrigger)),
               stateInternals,
               timerInternals,
               new OutputWindowedValueToBundle<>(bundle),
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java
new file mode 100644
index 000000000000..9928cb0d36b7
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.direct;
+
+import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.util.ApiSurface;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** API surface verification for {@link org.apache.beam.runners.direct}. */
+@RunWith(JUnit4.class)
+public class DirectRunnerApiSurfaceTest {
+  @Test
+  public void testDirectRunnerApiSurface() throws Exception {
+    // The DirectRunner can expose the Core SDK, anything exposed by the Core SDK, and itself
+    @SuppressWarnings("unchecked")
+    final Set<String> allowed =
+        ImmutableSet.of("org.apache.beam.sdk", "org.apache.beam.runners.direct", "org.joda.time");
+
+    final Package thisPackage = getClass().getPackage();
+    final ClassLoader thisClassLoader = getClass().getClassLoader();
+    ApiSurface apiSurface =
+        ApiSurface.ofPackage(thisPackage, thisClassLoader)
+            .pruningPattern("org[.]apache[.]beam[.].*Test.*")
+            .pruningPattern("org[.]apache[.]beam[.].*IT")
+            .pruningClass(Nullable.class)
+            .pruningPattern("java[.]io.*")
+            .pruningPattern("java[.]lang.*")
+            .pruningPattern("java[.]util.*");
+
+    assertThat(apiSurface, containsOnlyPackages(allowed));
+  }
+}

From b7b56022dac11e5fb344c93b6d8cbf49c4babd44 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 9 May 2017 11:41:56 -0700
Subject: [PATCH 146/387] Remove old imports meant for backwards compatibility

---
 sdks/python/apache_beam/io/gcp/bigquery.py    |  8 ++-----
 sdks/python/apache_beam/io/iobase.py          |  5 -----
 sdks/python/apache_beam/pvalue.py             |  7 -------
 .../runners/dataflow/internal/dependency.py   |  2 +-
 .../runners/direct/watermark_manager.py       |  4 ++--
 .../portability/maptask_executor_runner.py    |  3 ++-
 .../python/apache_beam/transforms/timeutil.py | 10 ---------
 sdks/python/apache_beam/transforms/trigger.py |  4 ++--
 sdks/python/apache_beam/transforms/window.py  | 12 +++++------
 .../apache_beam/transforms/window_test.py     |  4 ++--
 .../apache_beam/utils/pipeline_options.py     | 21 -------------------
 .../apache_beam/utils/value_provider.py       | 21 -------------------
 .../apache_beam/utils/windowed_value_test.py  |  2 +-
 13 files changed, 18 insertions(+), 85 deletions(-)
 delete mode 100644 sdks/python/apache_beam/utils/pipeline_options.py
 delete mode 100644 sdks/python/apache_beam/utils/value_provider.py

diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py
index 1ed28a82b668..201c798910fc 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery.py
@@ -1120,12 +1120,8 @@ def convert_row_to_dict(self, row, schema):
         value = cell['v'] if 'v' in cell else None
       if field.mode == 'REPEATED':
         if value is None:
-          # We receive 'None' for repeated fields without any values when
-          # 'flatten_results' is 'False'.
-          # When 'flatten_results' is 'True', we receive individual values
-          # instead of a list of values hence we do not hit this condition.
-          # We return an empty list here instead of 'None' to be consistent with
-          # other runners and to be backwards compatible to users.
+          # Ideally this should never happen as repeated fields default to
+          # returning an empty list
           result[field.name] = []
         else:
           result[field.name] = [self._convert_cell_value_to_dict(x['v'], field)
diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py
index d47ef5b19cb3..e8ffb7270640 100644
--- a/sdks/python/apache_beam/io/iobase.py
+++ b/sdks/python/apache_beam/io/iobase.py
@@ -979,8 +979,3 @@ def process(self, element):
     if self.counter >= self.count:
       self.counter -= self.count
     yield self.counter, element
-
-
-# For backwards compatibility.
-# pylint: disable=wrong-import-position
-from apache_beam.runners.dataflow.native_io.iobase import *
diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py
index 2242c5a8246f..fa91fe30de02 100644
--- a/sdks/python/apache_beam/pvalue.py
+++ b/sdks/python/apache_beam/pvalue.py
@@ -393,13 +393,6 @@ def _from_runtime_iterable(it, options):
     return dict(it)
 
 
-# For backwards compatibility with worker code.
-SingletonPCollectionView = AsSingleton
-IterablePCollectionView = AsIter
-ListPCollectionView = AsList
-DictPCollectionView = AsDict
-
-
 class EmptySideInput(object):
   """Value indicating when a singleton side input was empty.
 
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index 186ad3f21a77..63e593ce58d1 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -72,7 +72,7 @@
 
 # Update this version to the next version whenever there is a change that will
 # require changes to the execution environment.
-BEAM_CONTAINER_VERSION = 'beam-0.7.0-20170505'
+BEAM_CONTAINER_VERSION = '2.0.0'
 
 # Standard file names used for staging files.
 WORKFLOW_TARBALL_FILE = 'workflow.tar.gz'
diff --git a/sdks/python/apache_beam/runners/direct/watermark_manager.py b/sdks/python/apache_beam/runners/direct/watermark_manager.py
index 19d9085aab3c..7793478a85a8 100644
--- a/sdks/python/apache_beam/runners/direct/watermark_manager.py
+++ b/sdks/python/apache_beam/runners/direct/watermark_manager.py
@@ -23,8 +23,8 @@
 
 from apache_beam import pipeline
 from apache_beam import pvalue
-from apache_beam.transforms.timeutil import MAX_TIMESTAMP
-from apache_beam.transforms.timeutil import MIN_TIMESTAMP
+from apache_beam.utils.timestamp import MAX_TIMESTAMP
+from apache_beam.utils.timestamp import MIN_TIMESTAMP
 
 
 class WatermarkManager(object):
diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
index c07f27ff0de0..3e0878002581 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
@@ -30,6 +30,7 @@
 from apache_beam.runners import DataflowRunner
 from apache_beam.runners.dataflow.internal.dependency import _dependency_file_copy
 from apache_beam.runners.dataflow.internal.names import PropertyNames
+from apache_beam.runners.dataflow.native_io.iobase import NativeSource
 from apache_beam.runners.runner import PipelineResult
 from apache_beam.runners.runner import PipelineRunner
 from apache_beam.runners.runner import PipelineState
@@ -144,7 +145,7 @@ def run_Read(self, transform_node):
 
   def _run_read_from(self, transform_node, source):
     """Used when this operation is the result of reading source."""
-    if not isinstance(source, iobase.NativeSource):
+    if not isinstance(source, NativeSource):
       source = iobase.SourceBundle(1.0, source, None, None)
     output = transform_node.outputs[None]
     element_coder = self._get_coder(output)
diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py
index 435d41b65e85..ba4ef36d548a 100644
--- a/sdks/python/apache_beam/transforms/timeutil.py
+++ b/sdks/python/apache_beam/transforms/timeutil.py
@@ -23,16 +23,6 @@
 from abc import abstractmethod
 
 
-# For backwards compatibility.
-# TODO(robertwb): Remove.
-# pylint: disable=unused-import
-from apache_beam.utils.timestamp import Duration
-from apache_beam.utils.timestamp import MAX_TIMESTAMP
-from apache_beam.utils.timestamp import MIN_TIMESTAMP
-from apache_beam.utils.timestamp import Timestamp
-# pylint: enable=unused-import
-
-
 class TimeDomain(object):
   """Time domain for streaming timers."""
 
diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py
index bcb9dd307de1..97240d392d2c 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -28,14 +28,14 @@
 from apache_beam.coders import observable
 from apache_beam.transforms import combiners
 from apache_beam.transforms import core
-from apache_beam.transforms.timeutil import MAX_TIMESTAMP
-from apache_beam.transforms.timeutil import MIN_TIMESTAMP
 from apache_beam.transforms.timeutil import TimeDomain
 from apache_beam.transforms.window import GlobalWindow
 from apache_beam.transforms.window import TimestampCombiner
 from apache_beam.transforms.window import WindowedValue
 from apache_beam.transforms.window import WindowFn
 from apache_beam.runners.api import beam_runner_api_pb2
+from apache_beam.utils.timestamp import MAX_TIMESTAMP
+from apache_beam.utils.timestamp import MIN_TIMESTAMP
 
 
 class AccumulationMode(object):
diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py
index 44a5a267d8dc..29994c0b208a 100644
--- a/sdks/python/apache_beam/transforms/window.py
+++ b/sdks/python/apache_beam/transforms/window.py
@@ -56,13 +56,13 @@
 from apache_beam import coders
 from apache_beam.runners.api import beam_runner_api_pb2
 from apache_beam.transforms import timeutil
-from apache_beam.transforms.timeutil import Duration
-from apache_beam.transforms.timeutil import MAX_TIMESTAMP
-from apache_beam.transforms.timeutil import MIN_TIMESTAMP
-from apache_beam.transforms.timeutil import Timestamp
-from apache_beam.utils.windowed_value import WindowedValue
 from apache_beam.utils import proto_utils
 from apache_beam.utils import urns
+from apache_beam.utils.timestamp import Duration
+from apache_beam.utils.timestamp import MAX_TIMESTAMP
+from apache_beam.utils.timestamp import MIN_TIMESTAMP
+from apache_beam.utils.timestamp import Timestamp
+from apache_beam.utils.windowed_value import WindowedValue
 
 
 # TODO(ccy): revisit naming and semantics once Java Apache Beam finalizes their
@@ -414,7 +414,7 @@ def get_window_coder(self):
 
   def merge(self, merge_context):
     to_merge = []
-    end = timeutil.MIN_TIMESTAMP
+    end = MIN_TIMESTAMP
     for w in sorted(merge_context.windows, key=lambda w: w.start):
       if to_merge:
         if end > w.start:
diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py
index 0f613d788413..2d2b03d90ed8 100644
--- a/sdks/python/apache_beam/transforms/window_test.py
+++ b/sdks/python/apache_beam/transforms/window_test.py
@@ -29,8 +29,6 @@
 from apache_beam.transforms import Map
 from apache_beam.transforms import WindowInto
 from apache_beam.transforms.core import Windowing
-from apache_beam.transforms.timeutil import MAX_TIMESTAMP
-from apache_beam.transforms.timeutil import MIN_TIMESTAMP
 from apache_beam.transforms.trigger import AccumulationMode
 from apache_beam.transforms.trigger import AfterCount
 from apache_beam.transforms.util import assert_that, equal_to
@@ -44,6 +42,8 @@
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.transforms.window import WindowedValue
 from apache_beam.transforms.window import WindowFn
+from apache_beam.utils.timestamp import MAX_TIMESTAMP
+from apache_beam.utils.timestamp import MIN_TIMESTAMP
 
 
 def context(element, timestamp):
diff --git a/sdks/python/apache_beam/utils/pipeline_options.py b/sdks/python/apache_beam/utils/pipeline_options.py
deleted file mode 100644
index 51a17ed4cd53..000000000000
--- a/sdks/python/apache_beam/utils/pipeline_options.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""For backwards compatibility"""
-
-# pylint: disable=unused-import
-from apache_beam.options.pipeline_options import *
diff --git a/sdks/python/apache_beam/utils/value_provider.py b/sdks/python/apache_beam/utils/value_provider.py
deleted file mode 100644
index 46f39aef7671..000000000000
--- a/sdks/python/apache_beam/utils/value_provider.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""For backwards compatibility"""
-
-# pylint: disable=unused-import
-from apache_beam.options.value_provider import *
diff --git a/sdks/python/apache_beam/utils/windowed_value_test.py b/sdks/python/apache_beam/utils/windowed_value_test.py
index f2574109eed9..7883d1dfe857 100644
--- a/sdks/python/apache_beam/utils/windowed_value_test.py
+++ b/sdks/python/apache_beam/utils/windowed_value_test.py
@@ -22,7 +22,7 @@
 import unittest
 
 from apache_beam.utils import windowed_value
-from apache_beam.transforms.timeutil import Timestamp
+from apache_beam.utils.timestamp import Timestamp
 
 
 class WindowedValueTest(unittest.TestCase):

From d3ff8da7311572e93701bd94aeb9ed93abb0002e Mon Sep 17 00:00:00 2001
From: Raghu Angadi <rangadi@google.com>
Date: Mon, 8 May 2017 17:14:37 -0700
Subject: [PATCH 147/387] KafkaIO API clean up

Removes withKeyCoder() and withValueCoder() methods. Their meaning changed
when KafkaIO added support for Deserializers. The coders can be explicitly
specified using withKeyDeserializerAndCoder(), likewise for value.

This makes it explicit to the user that Deserializer is still required
and JavaDoc explains why/when both are required.
---
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 287 ++++++------------
 .../CoderBasedKafkaDeserializer.java          |  70 -----
 .../CoderBasedKafkaSerializer.java            |  72 -----
 .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 112 +------
 4 files changed, 97 insertions(+), 444 deletions(-)
 delete mode 100644 sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java
 delete mode 100644 sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java

diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index e21945fd7f6d..cb31ea236c58 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -67,8 +67,6 @@
 import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
 import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader;
 import org.apache.beam.sdk.io.kafka.KafkaCheckpointMark.PartitionMark;
-import org.apache.beam.sdk.io.kafka.serialization.CoderBasedKafkaDeserializer;
-import org.apache.beam.sdk.io.kafka.serialization.CoderBasedKafkaSerializer;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Gauge;
 import org.apache.beam.sdk.metrics.SinkMetrics;
@@ -158,49 +156,14 @@
  * }</pre>
  *
  * <p>Kafka provides deserializers for common types in
- * {@link org.apache.kafka.common.serialization}.
- *
- * <p>To read Avro data, {@code fromAvro} can be used. This does not require manually specifying
- * a {@link Coder} or {@link Deserializer}.
- *
- * <p>It's also possible to deserialize data using a Beam {@link Coder} via
- * {@link #readWithCoders(Coder, Coder)}, though this is discouraged because the particular
- * binary format is not guaranteed by coders. However, this can be useful
- * when exchanging data with a Beam pipeline that uses the same coder:
- *
- * <pre>{@code
- *
- *  pipeline
- *    .apply(KafkaIO.<MyKey, MyValue>readWithCoders(MyKeyCoder.of(), MyValueCoder.of())
- *       .withBootstrapServers("broker_1:9092,broker_2:9092")
- *       .withTopic("my_topic")
- *    )
- *    ...
- * }</pre>
- *
- * <p>In most cases, you don't need to specify {@link Coder} for key and value in the resulting
+ * {@link org.apache.kafka.common.serialization}. In addition to deserializers, Beam runners need
+ * {@link Coder} to materialize key and value objects if necessary.
+ * In most cases, you don't need to specify {@link Coder} for key and value in the resulting
  * collection because the coders are inferred from deserializer types. However, in cases when
- * coder inference fails, they can be specified manually using {@link Read#withKeyCoder} and
- * {@link Read#withValueCoder}. Note that the payloads of Kafka messages is interpreted using
- * key and value <i>deserializers</i>; coders are a Beam implementation detail to help runners
- * materialize the data for intermediate storage if necessary.
- *
- * <pre>{@code
- *
- *  pipeline
- *    .apply(KafkaIO.<Long, Foo>read()
- *       .withBootstrapServers("broker_1:9092,broker_2:9092")
- *       .withTopic("my_topic")
- *
- *       // infer coder from deserializer
- *       .withKeyDeserializer(LongDeserializer.class)
- *
- *       // explicitly specify coder
- *       .withValueDeserializer(FooDeserializer.class)
- *       .withValueCoder(FooCoder.of())
- *     )
- *     ...
- * }</pre>
+ * coder inference fails, they can be specified explicitly along with deserializers using
+ * {@link Read#withKeyDeserializerAndCoder(Class, Coder)} and
+ * {@link Read#withValueDeserializerAndCoder(Class, Coder)}. Note that Kafka messages are
+ * interpreted using key and value <i>deserializers</i>.
  *
  * <h3>Partition Assignment and Checkpointing</h3>
  * The Kafka partitions are evenly distributed among splits (workers).
@@ -257,11 +220,6 @@
  *    );
  * }</pre>
  *
- * <p>Same notes on coders vs. serializers apply as above for {@link Read}.
- *
- * <p>To write Avro data, {@code toAvro} can be used. This does not require specifying serializers
- * or coders.
- *
  * <h3>Advanced Kafka Configuration</h3>
  * KafkaIO allows setting most of the properties in {@link ConsumerConfig} for source or in
  * {@link ProducerConfig} for sink. E.g. if you would like to enable offset
@@ -279,46 +237,6 @@
 @Experimental
 public class KafkaIO {
 
-  /**
-   * Attempt to infer a {@link Coder} by extracting the type of the deserialized-class from the
-   * deserializer argument using the {@link Coder} registry.
-   */
-  @VisibleForTesting
-  static <T> NullableCoder<T> inferCoder(
-      CoderRegistry coderRegistry, Class<? extends Deserializer<T>> deserializer) {
-    checkNotNull(deserializer);
-
-    for (Type type : deserializer.getGenericInterfaces()) {
-      if (!(type instanceof ParameterizedType)) {
-        continue;
-      }
-
-      // This does not recurse: we will not infer from a class that extends
-      // a class that extends Deserializer<T>.
-      ParameterizedType parameterizedType = (ParameterizedType) type;
-
-      if (parameterizedType.getRawType() == Deserializer.class) {
-        Type parameter = parameterizedType.getActualTypeArguments()[0];
-
-        @SuppressWarnings("unchecked")
-        Class<T> clazz = (Class<T>) parameter;
-
-        try {
-          return NullableCoder.of(coderRegistry.getCoder(clazz));
-        } catch (CannotProvideCoderException e) {
-          throw new RuntimeException(
-                  String.format("Unable to automatically infer a Coder for "
-                                + "the Kafka Deserializer %s: no coder registered for type %s",
-                                deserializer, clazz));
-        }
-      }
-    }
-
-    throw new RuntimeException(
-            String.format("Could not extract the Kafaka Deserializer type from %s",
-                          deserializer));
-  }
-
   /**
    * Creates an uninitialized {@link Read} {@link PTransform}. Before use, basic Kafka
    * configuration should set with {@link Read#withBootstrapServers(String)} and
@@ -353,44 +271,6 @@ public static <K, V> Read<K, V> read() {
         .build();
   }
 
-  /**
-   * Creates an uninitialized {@link Read} {@link PTransform}, using Kafka {@link Deserializer}s
-   * based on {@link Coder} instances.
-   */
-  @SuppressWarnings("unchecked")
-  public static <K, V> Read<K, V> readWithCoders(Coder<K> keyCoder, Coder<V> valueCoder) {
-    // Kafka constructs deserializers directly. Pass coder through consumer
-    // configuration.
-    ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
-    Map<String, Object> config = builder
-            .putAll(Read.DEFAULT_CONSUMER_PROPERTIES)
-            .put(CoderBasedKafkaDeserializer.configForKeyDeserializer(), keyCoder)
-            .put(CoderBasedKafkaDeserializer.configForValueDeserializer(), valueCoder)
-            .build();
-
-    return new AutoValue_KafkaIO_Read.Builder<K, V>()
-            .setTopics(new ArrayList<String>())
-            .setTopicPartitions(new ArrayList<TopicPartition>())
-            .setKeyCoder(keyCoder)
-            .setValueCoder(valueCoder)
-            .setKeyDeserializer((Class) CoderBasedKafkaDeserializer.class)
-            .setValueDeserializer((Class) CoderBasedKafkaDeserializer.class)
-            .setConsumerFactoryFn(Read.KAFKA_CONSUMER_FACTORY_FN)
-            .setConsumerConfig(config)
-            .setMaxNumRecords(Long.MAX_VALUE)
-            .build();
-  }
-
-  /**
-   * Creates an uninitialized {@link Read} {@link PTransform}, using Kafka {@link Deserializer}s
-   * based on {@link AvroCoder}. This reads data in the Avro binary format directly without using
-   * an Avro object container.
-   */
-  @SuppressWarnings("unchecked")
-  public static <K, V> Read<K, V> fromAvro(Class<K> keyClass, Class<V> valueClass) {
-    return readWithCoders(AvroCoder.of(keyClass), AvroCoder.of(valueClass));
-  }
-
   /**
    * Creates an uninitialized {@link Write} {@link PTransform}. Before use, Kafka configuration
    * should be set with {@link Write#withBootstrapServers(String)} and {@link Write#withTopic}
@@ -401,40 +281,6 @@ public static <K, V> Write<K, V> write() {
         .setProducerConfig(Write.DEFAULT_PRODUCER_PROPERTIES)
         .build();
   }
-  /**
-   * Creates an uninitialized {@link Write} {@link PTransform}, using Kafka {@link Serializer}s
-   * based on {@link Coder} instances.
-   */
-  @SuppressWarnings("unchecked")
-  public static <K, V> Write<K, V> writeWithCoders(Coder<K> keyCoder, Coder<V> valueCoder) {
-    // Kafka constructs serializers directly. Pass coder through consumer
-    // configuration.
-    ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
-    Map<String, Object> config = builder
-            .putAll(Write.DEFAULT_PRODUCER_PROPERTIES)
-            .put(CoderBasedKafkaSerializer.configForKeySerializer(), keyCoder)
-            .put(CoderBasedKafkaSerializer.configForValueSerializer(), valueCoder)
-            .build();
-
-    CoderBasedKafkaSerializer<K> keySerializer = new CoderBasedKafkaSerializer<K>();
-    CoderBasedKafkaSerializer<V> valueSerializer = new CoderBasedKafkaSerializer<V>();
-
-    return new AutoValue_KafkaIO_Write.Builder<K, V>()
-            .setProducerConfig(config)
-            .setKeySerializer((Class) CoderBasedKafkaSerializer.class)
-            .setValueSerializer((Class) CoderBasedKafkaSerializer.class)
-            .build();
-  }
-
-  /**
-   * Creates an uninitialized {@link Write} {@link PTransform}, using Kafka {@link Serializer}s
-   * based on {@link AvroCoder}. The coder writes Avro data directly without using an Avro object
-   * container.
-   */
-  @SuppressWarnings("unchecked")
-  public static <K, V> Write<K, V> toAvro(Class<K> keyClass, Class<V> valueClass) {
-    return writeWithCoders(AvroCoder.of(keyClass), AvroCoder.of(valueClass));
-  }
 
   ///////////////////////// Read Support \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
 
@@ -524,31 +370,48 @@ public Read<K, V> withTopicPartitions(List<TopicPartition> topicPartitions) {
     }
 
     /**
-     * Returns a new {@link Read} with a Kafka {@link Deserializer} for key bytes.
+     * Returns new {@link Read} with a Kafka {@link Deserializer} to interpret key bytes read
+     * from Kafka. In addition, Beam also needs a Coder to serialize and deserialize key objects
+     * at runtime. KafkaIO tries to infer coders for many of the common types (bytes, strings,
+     * integers, etc) based on Deserializer class. It might not be able to do so for some types.
+     * Please use {@link #withKeyDeserializerAndCoder(Class, Coder)} to provide the key coder
+     * explicitly.
+     *
+     * @see KafkaIO.Read#withKeyDeserializerAndCoder
      */
     public Read<K, V> withKeyDeserializer(Class<? extends Deserializer<K>> keyDeserializer) {
       return toBuilder().setKeyDeserializer(keyDeserializer).build();
     }
 
     /**
-     * Returns a new {@link Read} with a {@link Coder} for the key.
+     * Returns new {@link Read} with a Kafka {@link Deserializer} to deserializer key bytes read
+     * from Kafka and {@link Coder} for serializing and deserializing the keys objects at runtime.
      */
-    public Read<K, V> withKeyCoder(Coder<K> keyCoder) {
-      return toBuilder().setKeyCoder(keyCoder).build();
+    public Read<K, V> withKeyDeserializerAndCoder(Class<? extends Deserializer<K>> keyDeserializer,
+                                                  Coder<K> keyCoder) {
+
+      return toBuilder()
+          .setKeyDeserializer(keyDeserializer)
+          .setKeyCoder(keyCoder)
+          .build();
     }
 
-    /**
-     * Returns a new {@link Read} with a Kafka {@link Deserializer} for value bytes.
-     */
     public Read<K, V> withValueDeserializer(Class<? extends Deserializer<V>> valueDeserializer) {
       return toBuilder().setValueDeserializer(valueDeserializer).build();
     }
 
     /**
-     * Returns a new {@link Read} with a {@link Coder} for values.
+     * Returns new {@link Read} with a Kafka {@link Deserializer} to deserializer value bytes read
+     * from Kafka and {@link Coder} for serializing and deserializing the value objects at runtime.
      */
-    public Read<K, V> withValueCoder(Coder<V> valueCoder) {
-      return toBuilder().setValueCoder(valueCoder).build();
+    public Read<K, V> withValueDeserializerAndCoder(
+        Class<? extends Deserializer<V>> valueDeserializer,
+        Coder<V> valueCoder) {
+
+      return toBuilder()
+          .setValueDeserializer(valueDeserializer)
+          .setValueCoder(valueCoder)
+          .build();
     }
 
     /**
@@ -649,20 +512,23 @@ public PCollection<KafkaRecord<K, V>> expand(PBegin input) {
       Coder<K> keyCoder =
           checkNotNull(
               getKeyCoder() != null ? getKeyCoder() : inferCoder(registry, getKeyDeserializer()),
-              "Key coder must be inferable from input or set using readWithCoders");
+              "Key coder could not be inferred from key deserializer. Please provide"
+                  + "key coder explicitly using withKeyDeserializerAndCoder()");
 
       Coder<V> valueCoder =
           checkNotNull(
-              getValueCoder() != null
-                  ? getValueCoder()
+              getValueCoder() != null ? getValueCoder()
                   : inferCoder(registry, getValueDeserializer()),
-              "Value coder must be inferable from input or set using readWithCoders");
+              "Value coder could not be inferred from value deserializer. Please provide"
+                  + "value coder explicitly using withValueDeserializerAndCoder()");
 
       // Handles unbounded source to bounded conversion if maxNumRecords or maxReadTime is set.
       Unbounded<KafkaRecord<K, V>> unbounded =
-          org.apache.beam.sdk.io.Read.from(this
-                  .withKeyCoder(keyCoder)
-                  .withValueCoder(valueCoder)
+          org.apache.beam.sdk.io.Read.from(
+              toBuilder()
+                  .setKeyCoder(keyCoder)
+                  .setValueCoder(valueCoder)
+                  .build()
                   .makeSource());
 
       PTransform<PBegin, PCollection<KafkaRecord<K, V>>> transform = unbounded;
@@ -683,6 +549,7 @@ public PCollection<KafkaRecord<K, V>> expand(PBegin input) {
      */
     @VisibleForTesting
     UnboundedSource<KafkaRecord<K, V>, KafkaCheckpointMark> makeSource() {
+
       return new UnboundedKafkaSource<K, V>(this, -1);
     }
 
@@ -703,11 +570,9 @@ public OutT apply(KafkaRecord<KeyT, ValueT> record) {
      */
     private static final Map<String, String> IGNORED_CONSUMER_PROPERTIES = ImmutableMap.of(
         ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "Set keyDeserializer instead",
-        ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "Set valueDeserializer instead",
+        ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "Set valueDeserializer instead"
         // "group.id", "enable.auto.commit", "auto.commit.interval.ms" :
         //     lets allow these, applications can have better resume point for restarts.
-        CoderBasedKafkaDeserializer.configForKeyDeserializer(), "Use readWithCoders instead",
-        CoderBasedKafkaDeserializer.configForValueDeserializer(), "Use readWithCoders instead"
         );
 
     // set config defaults
@@ -756,7 +621,9 @@ public void populateDisplayData(DisplayData.Builder builder) {
       for (Map.Entry<String, Object> conf : getConsumerConfig().entrySet()) {
         String key = conf.getKey();
         if (!ignoredConsumerPropertiesKeys.contains(key)) {
-          builder.add(DisplayData.item(key, ValueProvider.StaticValueProvider.of(conf.getValue())));
+          Object value = DisplayData.inferType(conf.getValue()) != null
+              ? conf.getValue() : String.valueOf(conf.getValue());
+          builder.add(DisplayData.item(key, ValueProvider.StaticValueProvider.of(value)));
         }
       }
     }
@@ -1543,10 +1410,7 @@ public void validate(PipelineOptions options) {
      */
     private static final Map<String, String> IGNORED_PRODUCER_PROPERTIES = ImmutableMap.of(
         ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "Use withKeySerializer instead",
-        ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "Use withValueSerializer instead",
-
-        CoderBasedKafkaSerializer.configForKeySerializer(), "Use writeWithCoders instead",
-        CoderBasedKafkaSerializer.configForValueSerializer(), "Use writeWithCoders instead"
+        ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "Use withValueSerializer instead"
      );
 
     @Override
@@ -1557,7 +1421,9 @@ public void populateDisplayData(DisplayData.Builder builder) {
       for (Map.Entry<String, Object> conf : getProducerConfig().entrySet()) {
         String key = conf.getKey();
         if (!ignoredProducerPropertiesKeys.contains(key)) {
-          builder.add(DisplayData.item(key, ValueProvider.StaticValueProvider.of(conf.getValue())));
+          Object value = DisplayData.inferType(conf.getValue()) != null
+              ? conf.getValue() : String.valueOf(conf.getValue());
+          builder.add(DisplayData.item(key, ValueProvider.StaticValueProvider.of(value)));
         }
       }
     }
@@ -1658,14 +1524,6 @@ public void teardown() {
     KafkaWriter(Write<K, V> spec) {
       this.spec = spec;
 
-      // Set custom kafka serializers. We do not want to serialize user objects then pass the bytes
-      // to producer since key and value objects are used in Kafka Partitioner interface.
-      // This does not matter for default partitioner in Kafka as it uses just the serialized
-      // key bytes to pick a partition. But we don't want to limit use of custom partitions.
-      // We pass key and values objects the user writes directly Kafka and user supplied
-      // coders to serialize them are invoked inside CoderBasedKafkaSerializer.
-      // Use case : write all the events for a single session to same Kafka partition.
-
       this.producerConfig = new HashMap<>(spec.getProducerConfig());
 
       this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
@@ -1708,4 +1566,43 @@ public void onCompletion(RecordMetadata metadata, Exception exception) {
       }
     }
   }
+
+  /**
+   * Attempt to infer a {@link Coder} by extracting the type of the deserialized-class from the
+   * deserializer argument using the {@link Coder} registry.
+   */
+  @VisibleForTesting
+  static <T> NullableCoder<T> inferCoder(
+      CoderRegistry coderRegistry, Class<? extends Deserializer<T>> deserializer) {
+    checkNotNull(deserializer);
+
+    for (Type type : deserializer.getGenericInterfaces()) {
+      if (!(type instanceof ParameterizedType)) {
+        continue;
+      }
+
+      // This does not recurse: we will not infer from a class that extends
+      // a class that extends Deserializer<T>.
+      ParameterizedType parameterizedType = (ParameterizedType) type;
+
+      if (parameterizedType.getRawType() == Deserializer.class) {
+        Type parameter = parameterizedType.getActualTypeArguments()[0];
+
+        @SuppressWarnings("unchecked")
+        Class<T> clazz = (Class<T>) parameter;
+
+        try {
+          return NullableCoder.of(coderRegistry.getCoder(clazz));
+        } catch (CannotProvideCoderException e) {
+          throw new RuntimeException(
+              String.format("Unable to automatically infer a Coder for "
+                                + "the Kafka Deserializer %s: no coder registered for type %s",
+                            deserializer, clazz));
+        }
+      }
+    }
+
+    throw new RuntimeException(String.format(
+        "Could not extract the Kafka Deserializer type from %s", deserializer));
+  }
 }
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java
deleted file mode 100644
index a1655860b36a..000000000000
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaDeserializer.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.kafka.serialization;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.Map;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.kafka.common.serialization.Deserializer;
-
-/**
- * Implements a Kafka {@link Deserializer} with a {@link Coder}.
- *
- * <p>As Kafka instantiates serializers directly, the coder must be stored as serialized value in
- * the producer configuration map.
- */
-public class CoderBasedKafkaDeserializer<T> implements Deserializer<T> {
-  @SuppressWarnings("unchecked")
-  @Override
-  public void configure(Map<String, ?> configs, boolean isKey) {
-    String configKey = isKey ? configForKeyDeserializer() : configForValueDeserializer();
-    coder = (Coder<T>) configs.get(configKey);
-    checkNotNull(coder, "could not instantiate coder for Kafka deserialization");
-  }
-
-  @Override
-  public T deserialize(String topic, byte[] data) {
-    if (data == null) {
-      return null;
-    }
-
-    try {
-      return CoderUtils.decodeFromByteArray(coder, data);
-    } catch (CoderException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public void close() {}
-
-  public static String configForKeyDeserializer() {
-    return String.format(CoderBasedKafkaDeserializer.CONFIG_FORMAT, "key");
-  }
-
-  public static String configForValueDeserializer() {
-    return String.format(CoderBasedKafkaDeserializer.CONFIG_FORMAT, "value");
-  }
-
-  private Coder<T> coder = null;
-  private static final String CONFIG_FORMAT = "beam.coder.based.kafka.%s.deserializer";
-}
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java
deleted file mode 100644
index 84b617ee8ac4..000000000000
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/serialization/CoderBasedKafkaSerializer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.io.kafka.serialization;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.Map;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.kafka.common.serialization.Serializer;
-
-/**
- * Implements Kafka's {@link Serializer} with a {@link Coder}.
- *
- * <p>As Kafka instantiates serializers directly, the coder
- * must be stored as serialized value in the producer configuration map.
- */
-public class CoderBasedKafkaSerializer<T> implements Serializer<T> {
-  @SuppressWarnings("unchecked")
-  @Override
-  public void configure(Map<String, ?> configs, boolean isKey) {
-    String configKey = isKey ? configForKeySerializer() : configForValueSerializer();
-    coder = (Coder<T>) configs.get(configKey);
-    checkNotNull(coder, "could not instantiate coder for Kafka serialization");
-  }
-
-  @Override
-  public byte[] serialize(String topic, @Nullable T data) {
-    if (data == null) {
-      return null; // common for keys to be null
-    }
-
-    try {
-      return CoderUtils.encodeToByteArray(coder, data);
-    } catch (CoderException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public void close() {
-  }
-
-  public static String configForKeySerializer() {
-    return String.format(CoderBasedKafkaSerializer.CONFIG_FORMAT, "key");
-  }
-
-  public static String configForValueSerializer() {
-    return String.format(CoderBasedKafkaSerializer.CONFIG_FORMAT, "value");
-  }
-
-  private Coder<T> coder = null;
-  private static final String CONFIG_FORMAT = "beam.coder.based.kafka.%s.serializer";
-}
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
index ccbd3d6703d7..12b7c78fd8cd 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
@@ -43,10 +43,11 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.UnboundedSource;
@@ -264,35 +265,6 @@ private static KafkaIO.Read<Integer, Long> mkKafkaReadTransform(
     }
   }
 
-  /**
-   * Creates a consumer with two topics, with 10 partitions each.
-   * numElements are (round-robin) assigned all the 20 partitions.
-   * Coders are specified explicitly.
-   */
-  private static KafkaIO.Read<Integer, Long> mkKafkaReadTransformWithCoders(
-          int numElements,
-          @Nullable SerializableFunction<KV<Integer, Long>, Instant> timestampFn) {
-
-    List<String> topics = ImmutableList.of("topic_a", "topic_b");
-
-    KafkaIO.Read<Integer, Long> reader = KafkaIO
-            .<Integer, Long>readWithCoders(VarIntCoder.of(), VarLongCoder.of())
-            .withBootstrapServers("myServer1:9092,myServer2:9092")
-            .withTopics(topics)
-            .withConsumerFactoryFn(new ConsumerFactoryFn(
-                    topics, 10, numElements, OffsetResetStrategy.EARLIEST)) // 20 partitions
-            .withKeyDeserializer(IntegerDeserializer.class)
-            .withValueDeserializer(LongDeserializer.class)
-            .withMaxNumRecords(numElements);
-
-    if (timestampFn != null) {
-      return reader.withTimestampFn(timestampFn);
-    } else {
-      return reader;
-    }
-  }
-
-
   private static class AssertMultipleOf implements SerializableFunction<Iterable<Long>, Void> {
     private final int num;
 
@@ -342,19 +314,6 @@ public void testUnboundedSource() {
     p.run();
   }
 
-  @Test
-  public void testUnboundedSourceWithCoders() {
-    int numElements = 1000;
-
-    PCollection<Long> input = p
-            .apply(mkKafkaReadTransformWithCoders(numElements, new ValueAsTimestampFn())
-                    .withoutMetadata())
-            .apply(Values.<Long>create());
-
-    addCountingAsserts(input, numElements);
-    p.run();
-  }
-
   @Test
   public void testUnboundedSourceWithSingleTopic() {
     // same as testUnboundedSource, but with single topic
@@ -454,9 +413,9 @@ public void testUnboundedSourceSplits() throws Exception {
     // is used in the test.
     UnboundedSource<KafkaRecord<Integer, Long>, ?> initial =
         mkKafkaReadTransform(numElements, null)
-                .withKeyCoder(VarIntCoder.of())
-                .withValueCoder(VarLongCoder.of())
-                .makeSource();
+            .withKeyDeserializerAndCoder(IntegerDeserializer.class, BigEndianIntegerCoder.of())
+            .withValueDeserializerAndCoder(LongDeserializer.class, BigEndianLongCoder.of())
+            .makeSource();
 
     List<? extends UnboundedSource<KafkaRecord<Integer, Long>, ?>> splits =
         initial.split(numSplits, p.getOptions());
@@ -716,39 +675,6 @@ public void testSink() throws Exception {
     }
   }
 
-  @Test
-  public void testSinkWithCoders() throws Exception {
-    // Simply read from kafka source and write to kafka sink. Then verify the records
-    // are correctly published to mock kafka producer.
-
-    int numElements = 1000;
-
-    synchronized (MOCK_PRODUCER_LOCK) {
-
-      MOCK_PRODUCER.clear();
-
-      ProducerSendCompletionThread completionThread = new ProducerSendCompletionThread().start();
-
-      String topic = "test";
-
-      p
-              .apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn())
-                      .withoutMetadata())
-              .apply(KafkaIO.<Integer, Long>writeWithCoders(VarIntCoder.of(), VarLongCoder.of())
-                      .withBootstrapServers("none")
-                      .withTopic(topic)
-                      .withKeySerializer(IntegerSerializer.class)
-                      .withValueSerializer(LongSerializer.class)
-                      .withProducerFactoryFn(new ProducerFactoryFn()));
-
-      p.run();
-
-      completionThread.shutdown();
-
-      verifyProducerRecords(topic, numElements, false);
-    }
-  }
-
   @Test
   public void testValuesSink() throws Exception {
     // similar to testSink(), but use values()' interface.
@@ -839,19 +765,6 @@ public void testSourceDisplayData() {
     assertThat(displayData, hasDisplayItem("receive.buffer.bytes", 524288));
   }
 
-  @Test
-  public void testSourceDisplayDataWithCoders() {
-    KafkaIO.Read<Integer, Long> read = mkKafkaReadTransformWithCoders(10, null);
-
-    DisplayData displayData = DisplayData.from(read);
-
-    assertThat(displayData, hasDisplayItem("topics", "topic_a,topic_b"));
-    assertThat(displayData, hasDisplayItem("enable.auto.commit", false));
-    assertThat(displayData, hasDisplayItem("bootstrap.servers", "myServer1:9092,myServer2:9092"));
-    assertThat(displayData, hasDisplayItem("auto.offset.reset", "latest"));
-    assertThat(displayData, hasDisplayItem("receive.buffer.bytes", 524288));
-  }
-
   @Test
   public void testSourceWithExplicitPartitionsDisplayData() {
     KafkaIO.Read<byte[], Long> read = KafkaIO.<byte[], Long>read()
@@ -886,21 +799,6 @@ public void testSinkDisplayData() {
     assertThat(displayData, hasDisplayItem("bootstrap.servers", "myServerA:9092,myServerB:9092"));
     assertThat(displayData, hasDisplayItem("retries", 3));
   }
-  @Test
-  public void testSinkDisplayDataWithCoders() {
-    KafkaIO.Write<Integer, Long> write = KafkaIO
-            .<Integer, Long>writeWithCoders(VarIntCoder.of(), VarLongCoder.of())
-            .withBootstrapServers("myServerA:9092,myServerB:9092")
-            .withTopic("myTopic")
-            .withValueSerializer(LongSerializer.class)
-            .withProducerFactoryFn(new ProducerFactoryFn());
-
-    DisplayData displayData = DisplayData.from(write);
-
-    assertThat(displayData, hasDisplayItem("topic", "myTopic"));
-    assertThat(displayData, hasDisplayItem("bootstrap.servers", "myServerA:9092,myServerB:9092"));
-    assertThat(displayData, hasDisplayItem("retries", 3));
-  }
 
   // interface for testing coder inference
   private interface DummyInterface<T> {

From fdd10892df750604af0e68bddcbda1275ceac8ed Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Tue, 9 May 2017 12:25:15 -0700
Subject: [PATCH 148/387] KafkaIO javadoc fixups

---
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 88 +++++++++++--------
 1 file changed, 49 insertions(+), 39 deletions(-)

diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index cb31ea236c58..a1130fc15e3f 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -329,7 +329,7 @@ abstract Builder<K, V> setConsumerFactoryFn(
     }
 
     /**
-     * Returns a new {@link Read} with Kafka consumer pointing to {@code bootstrapServers}.
+     * Sets the bootstrap servers for the Kafka consumer.
      */
     public Read<K, V> withBootstrapServers(String bootstrapServers) {
       return updateConsumerProperties(
@@ -338,8 +338,9 @@ public Read<K, V> withBootstrapServers(String bootstrapServers) {
     }
 
     /**
-     * Returns a new {@link Read} that reads from the topic.
-     * See {@link UnboundedKafkaSource#split(int, PipelineOptions)} for description
+     * Sets the topic to read from.
+     *
+     * <p>See {@link UnboundedKafkaSource#split(int, PipelineOptions)} for description
      * of how the partitions are distributed among the splits.
      */
     public Read<K, V> withTopic(String topic) {
@@ -347,9 +348,10 @@ public Read<K, V> withTopic(String topic) {
     }
 
     /**
-     * Returns a new {@link Read} that reads from the topics. All the partitions from each
+     * Sets a list of topics to read from. All the partitions from each
      * of the topics are read.
-     * See {@link UnboundedKafkaSource#split(int, PipelineOptions)} for description
+     *
+     * <p>See {@link UnboundedKafkaSource#split(int, PipelineOptions)} for description
      * of how the partitions are distributed among the splits.
      */
     public Read<K, V> withTopics(List<String> topics) {
@@ -359,9 +361,10 @@ public Read<K, V> withTopics(List<String> topics) {
     }
 
     /**
-     * Returns a new {@link Read} that reads from the partitions. This allows reading only a subset
+     * Sets a list of partitions to read from. This allows reading only a subset
      * of partitions for one or more topics when (if ever) needed.
-     * See {@link UnboundedKafkaSource#split(int, PipelineOptions)} for description
+     *
+     * <p>See {@link UnboundedKafkaSource#split(int, PipelineOptions)} for description
      * of how the partitions are distributed among the splits.
      */
     public Read<K, V> withTopicPartitions(List<TopicPartition> topicPartitions) {
@@ -370,48 +373,51 @@ public Read<K, V> withTopicPartitions(List<TopicPartition> topicPartitions) {
     }
 
     /**
-     * Returns new {@link Read} with a Kafka {@link Deserializer} to interpret key bytes read
-     * from Kafka. In addition, Beam also needs a Coder to serialize and deserialize key objects
-     * at runtime. KafkaIO tries to infer coders for many of the common types (bytes, strings,
-     * integers, etc) based on Deserializer class. It might not be able to do so for some types.
-     * Please use {@link #withKeyDeserializerAndCoder(Class, Coder)} to provide the key coder
-     * explicitly.
+     * Sets a Kafka {@link Deserializer} to interpret key bytes read from Kafka.
      *
-     * @see KafkaIO.Read#withKeyDeserializerAndCoder
+     * <p>In addition, Beam also needs a {@link Coder} to serialize and deserialize key objects at
+     * runtime. KafkaIO tries to infer a coder for the key based on the {@link Deserializer} class,
+     * however in case that fails, you can use {@link #withKeyDeserializerAndCoder(Class, Coder)} to
+     * provide the key coder explicitly.
      */
     public Read<K, V> withKeyDeserializer(Class<? extends Deserializer<K>> keyDeserializer) {
       return toBuilder().setKeyDeserializer(keyDeserializer).build();
     }
 
     /**
-     * Returns new {@link Read} with a Kafka {@link Deserializer} to deserializer key bytes read
-     * from Kafka and {@link Coder} for serializing and deserializing the keys objects at runtime.
+     * Sets a Kafka {@link Deserializer} for interpreting key bytes read from Kafka along with a
+     * {@link Coder} for helping the Beam runner materialize key objects at runtime if necessary.
+     *
+     * <p>Use this method only if your pipeline doesn't work with plain {@link
+     * #withKeyDeserializer(Class)}.
      */
-    public Read<K, V> withKeyDeserializerAndCoder(Class<? extends Deserializer<K>> keyDeserializer,
-                                                  Coder<K> keyCoder) {
-
-      return toBuilder()
-          .setKeyDeserializer(keyDeserializer)
-          .setKeyCoder(keyCoder)
-          .build();
+    public Read<K, V> withKeyDeserializerAndCoder(
+        Class<? extends Deserializer<K>> keyDeserializer, Coder<K> keyCoder) {
+      return toBuilder().setKeyDeserializer(keyDeserializer).setKeyCoder(keyCoder).build();
     }
 
+    /**
+     * Sets a Kafka {@link Deserializer} to interpret value bytes read from Kafka.
+     *
+     * <p>In addition, Beam also needs a {@link Coder} to serialize and deserialize value objects at
+     * runtime. KafkaIO tries to infer a coder for the value based on the {@link Deserializer}
+     * class, however in case that fails, you can use {@link #withValueDeserializerAndCoder(Class,
+     * Coder)} to provide the value coder explicitly.
+     */
     public Read<K, V> withValueDeserializer(Class<? extends Deserializer<V>> valueDeserializer) {
       return toBuilder().setValueDeserializer(valueDeserializer).build();
     }
 
     /**
-     * Returns new {@link Read} with a Kafka {@link Deserializer} to deserializer value bytes read
-     * from Kafka and {@link Coder} for serializing and deserializing the value objects at runtime.
+     * Sets a Kafka {@link Deserializer} for interpreting value bytes read from Kafka along with a
+     * {@link Coder} for helping the Beam runner materialize value objects at runtime if necessary.
+     *
+     * <p>Use this method only if your pipeline doesn't work with plain {@link
+     * #withValueDeserializer(Class)}.
      */
     public Read<K, V> withValueDeserializerAndCoder(
-        Class<? extends Deserializer<V>> valueDeserializer,
-        Coder<V> valueCoder) {
-
-      return toBuilder()
-          .setValueDeserializer(valueDeserializer)
-          .setValueCoder(valueCoder)
-          .build();
+        Class<? extends Deserializer<V>> valueDeserializer, Coder<V> valueCoder) {
+      return toBuilder().setValueDeserializer(valueDeserializer).setValueCoder(valueCoder).build();
     }
 
     /**
@@ -1342,15 +1348,16 @@ public Write<K, V> withBootstrapServers(String bootstrapServers) {
     }
 
     /**
-     * Returns a new {@link Write} transform that writes to given topic.
+     * Sets the Kafka topic to write to.
      */
     public Write<K, V> withTopic(String topic) {
       return toBuilder().setTopic(topic).build();
     }
 
     /**
-     * Returns a new {@link Write} with {@link Serializer} for serializing key (if any) to bytes.
-     * A key is optional while writing to Kafka. Note when a key is set, its hash is used to
+     * Sets a {@link Serializer} for serializing key (if any) to bytes.
+     *
+     * <p>A key is optional while writing to Kafka. Note when a key is set, its hash is used to
      * determine partition in Kafka (see {@link ProducerRecord} for more details).
      */
     public Write<K, V> withKeySerializer(Class<? extends Serializer<K>> keySerializer) {
@@ -1358,12 +1365,15 @@ public Write<K, V> withKeySerializer(Class<? extends Serializer<K>> keySerialize
     }
 
     /**
-     * Returns a new {@link Write} with {@link Serializer} for serializing value to bytes.
+     * Sets a {@link Serializer} for serializing value to bytes.
      */
     public Write<K, V> withValueSerializer(Class<? extends Serializer<V>> valueSerializer) {
       return toBuilder().setValueSerializer(valueSerializer).build();
     }
 
+    /**
+     * Adds the given producer properties, overriding old values of properties with the same key.
+     */
     public Write<K, V> updateProducerProperties(Map<String, Object> configUpdates) {
       Map<String, Object> config = updateKafkaProperties(getProducerConfig(),
           IGNORED_PRODUCER_PROPERTIES, configUpdates);
@@ -1371,7 +1381,7 @@ public Write<K, V> updateProducerProperties(Map<String, Object> configUpdates) {
     }
 
     /**
-     * Returns a new {@link Write} with a custom function to create Kafka producer. Primarily used
+     * Sets a custom function to create Kafka producer. Primarily used
      * for tests. Default is {@link KafkaProducer}
      */
     public Write<K, V> withProducerFactoryFn(
@@ -1380,8 +1390,8 @@ public Write<K, V> withProducerFactoryFn(
     }
 
     /**
-     * Returns a new transform that writes just the values to Kafka. This is useful for writing
-     * collections of values rather thank {@link KV}s.
+     * Writes just the values to Kafka. This is useful for writing collections of values rather
+     * thank {@link KV}s.
      */
     public PTransform<PCollection<V>, PDone> values() {
       return new KafkaValueWrite<>(toBuilder().build());

From 0bc87df3ef352b5677bf8ce6f2fa768a4fc0eadb Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Tue, 9 May 2017 11:29:20 -0700
Subject: [PATCH 149/387] [BEAM-2229] GcsFileSystem: handle empty files

And add tests
---
 .../extensions/gcp/storage/GcsFileSystem.java |  7 +++---
 .../gcp/storage/GcsFileSystemTest.java        | 22 +++++++++++++------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
index 38b8347a98bf..9052a5ac12cc 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.extensions.gcp.storage;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
@@ -241,10 +242,8 @@ private Metadata toMetadata(StorageObject storageObject) {
     Metadata.Builder ret = Metadata.builder()
         .setIsReadSeekEfficient(true)
         .setResourceId(GcsResourceId.fromGcsPath(GcsPath.fromObject(storageObject)));
-    BigInteger size = storageObject.getSize();
-    if (size != null) {
-      ret.setSizeBytes(size.longValue());
-    }
+    BigInteger size = firstNonNull(storageObject.getSize(), BigInteger.ZERO);
+    ret.setSizeBytes(size.longValue());
     return ret.build();
   }
 
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java
index 37ff9c807148..03194d28a77b 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java
@@ -35,6 +35,7 @@
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.Nullable;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.io.fs.MatchResult;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
@@ -62,13 +63,12 @@ public class GcsFileSystemTest {
   public transient ExpectedException thrown = ExpectedException.none();
   @Mock
   private GcsUtil mockGcsUtil;
-  private GcsOptions gcsOptions;
   private GcsFileSystem gcsFileSystem;
 
   @Before
   public void setUp() {
     MockitoAnnotations.initMocks(this);
-    gcsOptions = PipelineOptionsFactory.as(GcsOptions.class);
+    GcsOptions gcsOptions = PipelineOptionsFactory.as(GcsOptions.class);
     gcsOptions.setGcsUtil(mockGcsUtil);
     gcsFileSystem = new GcsFileSystem(gcsOptions);
   }
@@ -228,6 +228,8 @@ public void testMatchNonGlobs() throws Exception {
     // Files within the directory
     items.add(StorageObjectOrIOException.create(
         createStorageObject("gs://testbucket/testdirectory/file1name", 1L /* fileSize */)));
+    items.add(StorageObjectOrIOException.create(
+        createStorageObject("gs://testbucket/testdirectory/dir2name/", 0L /* fileSize */)));
     items.add(StorageObjectOrIOException.create(new FileNotFoundException()));
     items.add(StorageObjectOrIOException.create(new IOException()));
     items.add(StorageObjectOrIOException.create(
@@ -235,6 +237,7 @@ public void testMatchNonGlobs() throws Exception {
 
     List<GcsPath> gcsPaths = ImmutableList.of(
         GcsPath.fromUri("gs://testbucket/testdirectory/file1name"),
+        GcsPath.fromUri("gs://testbucket/testdirectory/dir2name/"),
         GcsPath.fromUri("gs://testbucket/testdirectory/file2name"),
         GcsPath.fromUri("gs://testbucket/testdirectory/file3name"),
         GcsPath.fromUri("gs://testbucket/testdirectory/file4name"));
@@ -242,23 +245,28 @@ public void testMatchNonGlobs() throws Exception {
     when(mockGcsUtil.getObjects(eq(gcsPaths))).thenReturn(items);
     List<MatchResult> matchResults = gcsFileSystem.matchNonGlobs(gcsPaths);
 
-    assertEquals(4, matchResults.size());
+    assertEquals(5, matchResults.size());
     assertThat(
         ImmutableList.of("gs://testbucket/testdirectory/file1name"),
         contains(toFilenames(matchResults.get(0)).toArray()));
-    assertEquals(Status.NOT_FOUND, matchResults.get(1).status());
-    assertEquals(Status.ERROR, matchResults.get(2).status());
+    assertThat(
+        ImmutableList.of("gs://testbucket/testdirectory/dir2name/"),
+        contains(toFilenames(matchResults.get(1)).toArray()));
+    assertEquals(Status.NOT_FOUND, matchResults.get(2).status());
+    assertEquals(Status.ERROR, matchResults.get(3).status());
     assertThat(
         ImmutableList.of("gs://testbucket/testdirectory/file4name"),
-        contains(toFilenames(matchResults.get(3)).toArray()));
+        contains(toFilenames(matchResults.get(4)).toArray()));
   }
 
   private StorageObject createStorageObject(String gcsFilename, long fileSize) {
     GcsPath gcsPath = GcsPath.fromUri(gcsFilename);
+    // Google APIs will use null for empty files.
+    @Nullable BigInteger size = (fileSize == 0) ? null : BigInteger.valueOf(fileSize);
     return new StorageObject()
         .setBucket(gcsPath.getBucket())
         .setName(gcsPath.getObject())
-        .setSize(BigInteger.valueOf(fileSize));
+        .setSize(size);
   }
 
   private List<String> toFilenames(MatchResult matchResult) throws IOException {

From 07084adde3a399a9f8f4863329af2b987ce29a08 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 14:04:39 -0700
Subject: [PATCH 150/387] Make PCollectionViewWindow package private

grep -lr "^public " runners/direct-java/src/main/java

runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
---
 .../org/apache/beam/runners/direct/PCollectionViewWindow.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PCollectionViewWindow.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PCollectionViewWindow.java
index 7a7d8ff238d7..ef5fa843e064 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PCollectionViewWindow.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PCollectionViewWindow.java
@@ -28,7 +28,7 @@
  *
  * @param <T> the type of the underlying PCollectionView
  */
-public final class PCollectionViewWindow<T> {
+final class PCollectionViewWindow<T> {
 
   private final PCollectionView<T> view;
   private final BoundedWindow window;

From 849f12253973e8f7ff153a0ee4b11371cec5fc12 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 8 May 2017 21:28:37 -0700
Subject: [PATCH 151/387] Revise javadoc for sdk, state, options, annotations,
 window, (and misc)

---
 .../java/org/apache/beam/sdk/Pipeline.java    | 57 +++++-------
 .../org/apache/beam/sdk/PipelineResult.java   | 11 ++-
 .../org/apache/beam/sdk/PipelineRunner.java   | 10 +--
 .../beam/sdk/annotations/Experimental.java    | 25 +++---
 .../apache/beam/sdk/annotations/Internal.java |  6 +-
 .../apache/beam/sdk/coders/AtomicCoder.java   |  6 +-
 .../java/org/apache/beam/sdk/io/Read.java     |  6 +-
 .../DefaultPipelineOptionsRegistrar.java      |  3 +-
 .../beam/sdk/options/PipelineOptionSpec.java  |  4 +-
 .../beam/sdk/options/ValueProvider.java       | 15 ++--
 .../org/apache/beam/sdk/state/BagState.java   |  6 +-
 .../apache/beam/sdk/state/CombiningState.java | 23 +++--
 .../apache/beam/sdk/state/GroupingState.java  | 17 ++--
 .../org/apache/beam/sdk/state/MapState.java   | 27 ++----
 .../apache/beam/sdk/state/ReadableState.java  | 16 ++--
 .../org/apache/beam/sdk/state/SetState.java   | 14 ++-
 .../java/org/apache/beam/sdk/state/State.java |  6 +-
 .../apache/beam/sdk/state/StateBinder.java    |  1 -
 .../apache/beam/sdk/state/StateContext.java   |  1 -
 .../apache/beam/sdk/state/StateContexts.java  |  5 +-
 .../org/apache/beam/sdk/state/StateSpecs.java | 86 ++++++++++++++-----
 .../org/apache/beam/sdk/state/TimeDomain.java |  8 +-
 .../java/org/apache/beam/sdk/state/Timer.java | 12 ++-
 .../org/apache/beam/sdk/state/TimerSpecs.java |  4 +-
 .../org/apache/beam/sdk/state/ValueState.java |  8 +-
 .../beam/sdk/state/WatermarkHoldState.java    |  4 +-
 .../sdk/transforms/windowing/AfterAll.java    |  4 +-
 .../sdk/transforms/windowing/AfterFirst.java  |  3 +-
 .../sdk/transforms/windowing/AfterPane.java   |  3 +-
 .../windowing/AfterProcessingTime.java        |  4 +-
 .../transforms/windowing/DefaultTrigger.java  |  8 +-
 .../transforms/windowing/GlobalWindows.java   |  5 +-
 .../transforms/windowing/InvalidWindows.java  |  2 +-
 .../MergeOverlappingIntervalWindows.java      |  6 +-
 .../beam/sdk/transforms/windowing/Never.java  |  2 +-
 .../windowing/NonMergingWindowFn.java         |  2 +-
 .../windowing/OrFinallyTrigger.java           |  5 +-
 .../sdk/transforms/windowing/Repeatedly.java  |  4 +-
 .../sdk/transforms/windowing/Sessions.java    | 13 +--
 .../windowing/TimestampCombiner.java          |  5 ++
 .../beam/sdk/transforms/windowing/Window.java | 12 +--
 .../sdk/transforms/windowing/WindowFn.java    | 10 +--
 .../transforms/windowing/WindowMappingFn.java | 12 ++-
 .../org/apache/beam/sdk/values/PBegin.java    | 26 +++---
 44 files changed, 271 insertions(+), 236 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index f4da6ad9c1c2..83496a5302ad 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -53,19 +53,14 @@
 
 /**
  * A {@link Pipeline} manages a directed acyclic graph of {@link PTransform PTransforms}, and the
- * {@link PCollection PCollections} that the {@link PTransform}s consume and produce.
- *
- * <p>A {@link Pipeline} is initialized with a {@link PipelineRunner} that will later
- * execute the {@link Pipeline}.
- *
- * <p>{@link Pipeline Pipelines} are independent, so they can be constructed and executed
- * concurrently.
+ * {@link PCollection PCollections} that the {@link PTransform PTransforms} consume and produce.
  *
  * <p>Each {@link Pipeline} is self-contained and isolated from any other
  * {@link Pipeline}. The {@link PValue PValues} that are inputs and outputs of each of a
  * {@link Pipeline Pipeline's} {@link PTransform PTransforms} are also owned by that
  * {@link Pipeline}. A {@link PValue} owned by one {@link Pipeline} can be read only by
- * {@link PTransform PTransforms} also owned by that {@link Pipeline}.
+ * {@link PTransform PTransforms} also owned by that {@link Pipeline}. {@link Pipeline Pipelines}
+ * can safely be executed concurrently.
  *
  * <p>Here is a typical example of use:
  * <pre> {@code
@@ -130,9 +125,7 @@ public PipelineExecutionException(Throwable cause) {
   // Public operations.
 
   /**
-   * Constructs a pipeline from default options.
-   *
-   * @return The newly created pipeline.
+   * Constructs a pipeline from default {@link PipelineOptions}.
    */
   public static Pipeline create() {
     Pipeline pipeline = new Pipeline(PipelineOptionsFactory.create());
@@ -141,9 +134,7 @@ public static Pipeline create() {
   }
 
   /**
-   * Constructs a pipeline from the provided options.
-   *
-   * @return The newly created pipeline.
+   * Constructs a pipeline from the provided {@link PipelineOptions}.
    */
   public static Pipeline create(PipelineOptions options) {
     // TODO: fix runners that mutate PipelineOptions in this method, then remove this line
@@ -155,9 +146,8 @@ public static Pipeline create(PipelineOptions options) {
   }
 
   /**
-   * Returns a {@link PBegin} owned by this Pipeline.  This is useful
-   * as the input of a root PTransform such as {@link Read} or
-   * {@link Create}.
+   * Returns a {@link PBegin} owned by this Pipeline. This serves as the input of a root {@link
+   * PTransform} such as {@link Read} or {@link Create}.
    */
   public PBegin begin() {
     return PBegin.in(this);
@@ -175,12 +165,12 @@ public <OutputT extends POutput> OutputT apply(
   }
 
   /**
-   * Adds a root {@link PTransform}, such as {@link Read} or {@link Create},
-   * to this {@link Pipeline}.
+   * Adds a root {@link PTransform}, such as {@link Read} or {@link Create}, to this {@link
+   * Pipeline}.
    *
-   * <p>The node in the {@link Pipeline} graph will use the provided {@code name}.
-   * This name is used in various places, including the monitoring UI, logging,
-   * and to stably identify this node in the {@link Pipeline} graph upon update.
+   * <p>The node in the {@link Pipeline} graph will use the provided {@code name}. This name is used
+   * in various places, including the monitoring UI, logging, and to stably identify this node in
+   * the {@link Pipeline} graph upon update.
    *
    * <p>Alias for {@code begin().apply(name, root)}.
    */
@@ -284,19 +274,16 @@ public void visitPrimitiveTransform(Node node) {
   }
 
   /**
-   * Runs this {@link Pipeline} using the default {@link PipelineOptions} provided
-   * to {@link #create(PipelineOptions)}.
-   *
-   * <p>It is an error to call this method if the pipeline was created without
-   * a default set of options.
+   * Runs this {@link Pipeline} according to the {@link PipelineOptions} used to create the {@link
+   * Pipeline} via {@link #create(PipelineOptions)}.
    */
   public PipelineResult run() {
     return run(defaultOptions);
   }
 
   /**
-   * Runs this {@link Pipeline} using the given {@link PipelineOptions}, using the runner
-   * specified by the options.
+   * Runs this {@link Pipeline} using the given {@link PipelineOptions}, using the runner specified
+   * by the options.
    */
   public PipelineResult run(PipelineOptions options) {
     PipelineRunner runner = PipelineRunner.fromOptions(options);
@@ -316,9 +303,6 @@ public PipelineResult run(PipelineOptions options) {
   }
 
 
-  /////////////////////////////////////////////////////////////////////////////
-  // Below here are operations that aren't normally called by users.
-
   /**
    * Returns the {@link CoderRegistry} that this {@link Pipeline} uses.
    */
@@ -329,9 +313,14 @@ public CoderRegistry getCoderRegistry() {
     return coderRegistry;
   }
 
+  /////////////////////////////////////////////////////////////////////////////
+  // Below here are operations that aren't normally called by users.
+
   /**
-   * Sets the {@link CoderRegistry} that this {@link Pipeline} uses.
+   * @deprecated this should never be used - every {@link Pipeline} has a registry throughout its
+   *     lifetime.
    */
+  @Deprecated
   public void setCoderRegistry(CoderRegistry coderRegistry) {
     this.coderRegistry = coderRegistry;
   }
@@ -554,8 +543,6 @@ void validate(PipelineOptions options) {
   /**
    * Returns a unique name for a transform with the given prefix (from
    * enclosing transforms) and initial name.
-   *
-   * <p>For internal use only.
    */
   private String uniquifyInternal(String namePrefix, String origName) {
     String name = origName;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
index 7e78e6e5fedc..b60de63678e5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
@@ -25,6 +25,8 @@
 
 /**
  * Result of {@link Pipeline#run()}.
+ *
+ * <p>This is often a job handle to an underlying data processing engine.
  */
 public interface PipelineResult {
 
@@ -65,7 +67,12 @@ public interface PipelineResult {
 
   // TODO: method to retrieve error messages.
 
-  /** Named constants for common values for the job state. */
+  /**
+   * Possible job states, for both completed and ongoing jobs.
+   *
+   * <p>When determining if a job is still running, consult the {@link #isTerminal()} method rather
+   * than inspecting the precise state.
+   */
   enum State {
 
     /** The job state could not be obtained or was not specified. */
@@ -114,7 +121,7 @@ public final boolean hasReplacementJob() {
   }
 
   /**
-   * Return the object to access metrics from the pipeline.
+   * Returns the object to access metrics from the pipeline.
    *
    * @throws UnsupportedOperationException if the runner doesn't support retrieving metrics.
    */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
index 18e79eb7fa27..87705af22d6a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
@@ -25,15 +25,14 @@
 import org.apache.beam.sdk.util.InstanceBuilder;
 
 /**
- * A {@link PipelineRunner} can execute, translate, or otherwise process a
- * {@link Pipeline}.
+ * A {@link PipelineRunner} runs a {@link Pipeline}.
  *
- * @param <ResultT> the type of the result of {@link #run}.
+ * @param <ResultT> the type of the result of {@link #run}, often a handle to a running job.
  */
 public abstract class PipelineRunner<ResultT extends PipelineResult> {
 
   /**
-   * Constructs a runner from the provided options.
+   * Constructs a runner from the provided {@link PipelineOptions}.
    *
    * @return The newly created runner.
    */
@@ -55,7 +54,8 @@ public static PipelineRunner<? extends PipelineResult> fromOptions(PipelineOptio
   }
 
   /**
-   * Processes the given Pipeline, returning the results.
+   * Processes the given {@link Pipeline}, potentially asynchronously, returning a runner-specific
+   * type of result.
    */
   public abstract ResultT run(Pipeline pipeline);
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
index 2e3a711cddfa..ac02465b2f55 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
@@ -24,25 +24,20 @@
 import java.lang.annotation.Target;
 
 /**
- * Signifies that a public API (public class, method or field) is subject to
- * incompatible changes, or even removal, in a future release. An API bearing
- * this annotation is exempt from any compatibility guarantees made by its
- * containing library. Note that the presence of this annotation implies nothing
- * about the quality or performance of the API in question, only the fact that
- * it is not "API-frozen."
+ * Signifies that a public API (public class, method or field) is subject to incompatible changes,
+ * or even removal, in a future release.
  *
- * <p>It is generally safe for <i>applications</i> to depend on experimental
- * APIs, at the cost of some extra work during upgrades. However, it is
- * generally inadvisable for <i>libraries</i> (which get included on users'
- * class paths, outside the library developers' control) to do so.
+ * <p>Note that the presence of this annotation implies nothing about the quality or performance of
+ * the API in question, only the fact that the API or behavior may change in any way.
  */
 @Retention(RetentionPolicy.CLASS)
 @Target({
-    ElementType.ANNOTATION_TYPE,
-    ElementType.CONSTRUCTOR,
-    ElementType.FIELD,
-    ElementType.METHOD,
-    ElementType.TYPE})
+  ElementType.ANNOTATION_TYPE,
+  ElementType.CONSTRUCTOR,
+  ElementType.FIELD,
+  ElementType.METHOD,
+  ElementType.TYPE
+})
 @Documented
 public @interface Experimental {
   Kind value() default Kind.UNSPECIFIED;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Internal.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Internal.java
index c359ef6d1ddc..b0dccad62693 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Internal.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Internal.java
@@ -26,9 +26,9 @@
 
 /**
  * Signifies that a publicly accessible API (public class, method or field) is intended for internal
- * use and not for public consumption. Such an API is subject to incompatible changes or removal at
- * any time. An API bearing this annotation is exempt from any compatibility guarantees made by its
- * containing library.
+ * use only and not for public consumption.
+ *
+ * <p>Such an API is subject to incompatible changes or removal at any time.
  */
 @Retention(RetentionPolicy.CLASS)
 @Target({
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
index 043fe939f132..7bcd5327c014 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
@@ -22,12 +22,12 @@
 import java.util.List;
 
 /**
- * A {@link Coder} that has no component {@link Coder Coders} or other state.
+ * A {@link Coder} that has no component {@link Coder Coders} or other configuration.
  *
- * <p>Note that, unless the behavior is overridden, atomic coders are presumed to be deterministic.
+ * <p>Unless the behavior is overridden, atomic coders are presumed to be deterministic.
  *
  * <p>All atomic coders of the same class are considered to be equal to each other. As a result,
- * an {@link AtomicCoder} should have no associated state.
+ * an {@link AtomicCoder} should have no associated configuration (instance variables, etc).
  *
  * @param <T> the type of the values being transcoded
  */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
index dbd9b86758d6..a07fca89457d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java
@@ -49,15 +49,15 @@ public static <T> Bounded<T> from(BoundedSource<T> source) {
   }
 
   /**
-   * Returns a new {@code Read.Unbounded} {@code PTransform} reading from the given
-   * {@code UnboundedSource}.
+   * Returns a new {@link Read.Unbounded} {@link PTransform} reading from the given
+   * {@link UnboundedSource}.
    */
   public static <T> Unbounded<T> from(UnboundedSource<T, ?> source) {
     return new Unbounded<>(null, source);
   }
 
   /**
-   * Helper class for building {@code Read} transforms.
+   * Helper class for building {@link Read} transforms.
    */
   public static class Builder {
     private final String name;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java
index b0ce8121a9af..3375dc7f9338 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java
@@ -22,7 +22,8 @@
 import com.google.common.collect.ImmutableList;
 
 /**
- * A registrar containing the default SDK options.
+ * A {@link PipelineOptionsRegistrar} containing the {@link PipelineOptions} subclasses available by
+ * default.
  */
 @AutoService(PipelineOptionsRegistrar.class)
 public class DefaultPipelineOptionsRegistrar implements PipelineOptionsRegistrar {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
index 1220e6bc14e2..265c2adb993d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
@@ -21,9 +21,7 @@
 import com.google.auto.value.AutoValue;
 import java.lang.reflect.Method;
 
-/**
- * For internal use. Specification for an option defined in a {@link PipelineOptions} interface.
- */
+/** Specification for an option defined in a {@link PipelineOptions} interface. */
 @AutoValue
 abstract class PipelineOptionSpec {
   static PipelineOptionSpec of(Class<? extends PipelineOptions> clazz, String name, Method getter) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
index 030eed595939..c7f1e094559a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
@@ -40,13 +40,14 @@
 import java.lang.reflect.Proxy;
 import java.util.concurrent.ConcurrentHashMap;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 
 /**
- * {@link ValueProvider} is an interface which abstracts the notion of
- * fetching a value that may or may not be currently available.  This can be
- * used to parameterize transforms that only read values in at runtime, for
- * example.
+ * A {@link ValueProvider} abstracts the notion of fetching a value that may or may not be currently
+ * available.
+ *
+ * <p>This can be used to parameterize transforms that only read values in at runtime, for example.
  */
 @JsonSerialize(using = ValueProvider.Serializer.class)
 @JsonDeserialize(using = ValueProvider.Deserializer.class)
@@ -264,8 +265,9 @@ public String toString() {
   }
 
   /**
-   * Serializer for {@link ValueProvider}.
+   * <b>For internal use only; no backwards compatibility guarantees.</b>
    */
+  @Internal
   class Serializer extends JsonSerializer<ValueProvider<?>> {
     @Override
     public void serialize(ValueProvider<?> value, JsonGenerator jgen,
@@ -279,8 +281,9 @@ public void serialize(ValueProvider<?> value, JsonGenerator jgen,
   }
 
   /**
-   * Deserializer for {@link ValueProvider}, which handles type marshalling.
+   * <b>For internal use only; no backwards compatibility guarantees.</b>
    */
+  @Internal
   class Deserializer extends JsonDeserializer<ValueProvider<?>>
     implements ContextualDeserializer {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
index 10fba05d1fbb..76d3e32a635c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/BagState.java
@@ -21,7 +21,11 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
- * State containing a bag values. Items can be added to the bag and the contents read out.
+ * A {@link ReadableState} cell containing a bag of values. Items can be added to the bag and the
+ * contents read out.
+ *
+ * <p>Implementations of this form of state are expected to implement {@link #add} efficiently, not
+ * via a sequence of read-modify-write.
  *
  * @param <T> The type of elements in the bag.
  */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
index ddda25519e3f..94a36d3c4976 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/CombiningState.java
@@ -22,33 +22,32 @@
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
 
 /**
- * State for a single value that is managed by a {@link CombineFn}. This is an internal extension
- * to {@link GroupingState} that includes the {@code AccumT} type.
+ * A {@link ReadableState} cell defined by a {@link CombineFn}, accepting multiple input values,
+ * combining them as specified into accumulators, and producing a single output value.
+ *
+ * <p>Implementations of this form of state are expected to implement {@link #add} efficiently, not
+ * via a sequence of read-modify-write.
  *
  * @param <InputT> the type of values added to the state
  * @param <AccumT> the type of accumulator
  * @param <OutputT> the type of value extracted from the state
  */
 @Experimental(Kind.STATE)
-public interface CombiningState<InputT, AccumT, OutputT>
-    extends GroupingState<InputT, OutputT> {
+public interface CombiningState<InputT, AccumT, OutputT> extends GroupingState<InputT, OutputT> {
 
   /**
-   * Read the merged accumulator for this combining value. It is implied that reading the
-   * state involes reading the accumulator, so {@link #readLater} is sufficient to prefetch for
-   * this.
+   * Read the merged accumulator for this state cell. It is implied that reading the state involves
+   * reading the accumulator, so {@link #readLater} is sufficient to prefetch for this.
    */
   AccumT getAccum();
 
   /**
-   * Add an accumulator to this combining value. Depending on implementation this may immediately
-   * merge it with the previous accumulator, or may buffer this accumulator for a future merge.
+   * Add an accumulator to this state cell. Depending on implementation this may immediately merge
+   * it with the previous accumulator, or may buffer this accumulator for a future merge.
    */
   void addAccum(AccumT accum);
 
-  /**
-   * Merge the given accumulators according to the underlying combiner.
-   */
+  /** Merge the given accumulators according to the underlying {@link CombineFn}. */
   AccumT mergeAccumulators(Iterable<AccumT> accumulators);
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
index d99ff25515cf..9c4c23e4cbd7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/GroupingState.java
@@ -19,25 +19,24 @@
 
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.GroupByKey;
 
 /**
- * State that combines multiple {@code InputT} values using a {@link CombineFn} to produce a single
- * {@code OutputT} value.
+ * A {@link ReadableState} cell that combines multiple input values and outputs a single value of a
+ * different type.
+ *
+ * <p>This generalizes {@link GroupByKey} and {@link Combine} styles of grouping.
  *
  * @param <InputT> the type of values added to the state
  * @param <OutputT> the type of value extracted from the state
  */
 @Experimental(Kind.STATE)
 public interface GroupingState<InputT, OutputT> extends ReadableState<OutputT>, State {
-  /**
-   * Add a value to the buffer.
-   */
+  /** Add a value to the buffer. */
   void add(InputT value);
 
-  /**
-   * Return true if this state is empty.
-   */
+  /** Return true if this state is empty. */
   ReadableState<Boolean> isEmpty();
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
index 649c3c7653d8..17ea33278c47 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/MapState.java
@@ -22,9 +22,10 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
- * An object that maps keys to values.
- * A map cannot contain duplicate keys;
- * each key can map to at most one value.
+ * A {@link ReadableState} cell mapping keys to values.
+ *
+ * <p>Implementations of this form of state are expected to implement map operations efficiently
+ * as supported by some associated backing key-value store.
  *
  * @param <K> the type of keys maintained by this map
  * @param <V> the type of mapped values
@@ -32,9 +33,7 @@
 @Experimental(Kind.STATE)
 public interface MapState<K, V> extends State {
 
-  /**
-   * Associates the specified value with the specified key in this state.
-   */
+  /** Associates the specified value with the specified key in this state. */
   void put(K key, V value);
 
   /**
@@ -48,9 +47,7 @@ public interface MapState<K, V> extends State {
    */
   ReadableState<V> putIfAbsent(K key, V value);
 
-  /**
-   * Removes the mapping for a key from this map if it is present.
-   */
+  /** Remove the mapping for a key from this map if it is present. */
   void remove(K key);
 
   /**
@@ -64,19 +61,13 @@ public interface MapState<K, V> extends State {
    */
   ReadableState<V> get(K key);
 
-  /**
-   * Returns a iterable view of the keys contained in this map.
-   */
+  /** Returns an {@link Iterable} over the keys contained in this map. */
   ReadableState<Iterable<K>> keys();
 
-  /**
-   * Returns a iterable view of the values contained in this map.
-   */
+  /** Returns an {@link Iterable} over the values contained in this map. */
   ReadableState<Iterable<V>> values();
 
-  /**
-   * Returns a iterable view of all key-values.
-   */
+  /** Returns an {@link Iterable} over the key-value pairs contained in this map. */
   ReadableState<Iterable<Map.Entry<K, V>>> entries();
 }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
index b29ab266a1ca..70703ce7efc2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ReadableState.java
@@ -21,10 +21,10 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
- * {@link State} that can be read via {@link #read()}.
+ * A {@link State} that can be read via {@link #read()}.
  *
- * <p>Use {@link #readLater()} for marking several states for prefetching. Runners
- * can potentially batch these into one read.
+ * <p>Use {@link #readLater()} for marking several states for prefetching. Runners can potentially
+ * batch these into one read.
  *
  * @param <T> The type of value returned by {@link #read}.
  */
@@ -33,17 +33,17 @@ public interface ReadableState<T> {
   /**
    * Read the current value, blocking until it is available.
    *
-   * <p>If there will be many calls to {@link #read} for different state in short succession,
-   * you should first call {@link #readLater} for all of them so the reads can potentially be
-   * batched (depending on the underlying implementation}.
+   * <p>If there will be many calls to {@link #read} for different state in short succession, you
+   * should first call {@link #readLater} for all of them so the reads can potentially be batched
+   * (depending on the underlying implementation}.
    */
   T read();
 
   /**
    * Indicate that the value will be read later.
    *
-   * <p>This allows an implementation to start an asynchronous prefetch or
-   * to include this state in the next batch of reads.
+   * <p>This allows an implementation to start an asynchronous prefetch or to include this state in
+   * the next batch of reads.
    *
    * @return this for convenient chaining
    */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
index cb9a0e6dd780..fd339b2778ea 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/SetState.java
@@ -21,16 +21,16 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
- * State containing no duplicate elements.
- * Items can be added to the set and the contents read out.
+ * A {@link ReadableState} cell containing a set of elements.
+ *
+ * <p>Implementations of this form of state are expected to implement set operations such as {@link
+ * #contains(Object)} efficiently, reading as little of the overall set as possible.
  *
  * @param <T> The type of elements in the set.
  */
 @Experimental(Kind.STATE)
 public interface SetState<T> extends GroupingState<T, Iterable<T>> {
-  /**
-   * Returns true if this set contains the specified element.
-   */
+  /** Returns true if this set contains the specified element. */
   ReadableState<Boolean> contains(T t);
 
   /**
@@ -39,9 +39,7 @@ public interface SetState<T> extends GroupingState<T, Iterable<T>> {
    */
   ReadableState<Boolean> addIfAbsent(T t);
 
-  /**
-   * Removes the specified element from this set if it is present.
-   */
+  /** Removes the specified element from this set if it is present. */
   void remove(T t);
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
index 0c0ca32f2ecc..63a9420d2962 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/State.java
@@ -21,10 +21,10 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
- * Base interface for all state locations.
+ * A state cell, supporting a {@link #clear()} operation.
  *
- * <p>Specific types of state add appropriate accessors for reading and writing values, see
- * {@link ValueState}, {@link BagState}, and {@link GroupingState}.
+ * <p>Specific types of state add appropriate accessors for reading and writing values, see {@link
+ * ValueState}, {@link BagState}, and {@link GroupingState}.
  */
 @Experimental(Kind.STATE)
 public interface State {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
index af834c3cdc91..29c7600ee0ca 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateBinder.java
@@ -24,7 +24,6 @@
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 
 /**
- * For internal use only; no backwards-compatibility guarantees.
  * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  *
  * <p>Visitor for binding a {@link StateSpec} and to the associated {@link State}.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
index b7dd8e2681cf..07bb05c124b4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContext.java
@@ -24,7 +24,6 @@
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
- * For internal use only; no backwards-compatibility guarantees.
  * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  */
 @Internal
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
index 2e21a27c1a6a..d61707940b3c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateContexts.java
@@ -22,10 +22,7 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.values.PCollectionView;
 
-/**
- * For internal use only; no backwards-compatibility guarantees.
- * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
- */
+/** <b><i>For internal use only; no backwards-compatibility guarantees.</i></b> */
 @Internal
 public class StateContexts {
   private static final StateContext<BoundedWindow> NULL_CONTEXT =
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
index 8a3c87e4c6a4..7b7138489997 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
@@ -31,9 +31,7 @@
 import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 
-/**
- * Static utility methods for creating {@link StateSpec} instances.
- */
+/** Static methods for working with {@link StateSpec StateSpecs}. */
 @Experimental(Kind.STATE)
 public class StateSpecs {
 
@@ -41,30 +39,50 @@ public class StateSpecs {
 
   private StateSpecs() {}
 
-  /** Create a simple state spec for values of type {@code T}. */
+  /**
+   * Create a {@link StateSpec} for a single value of type {@code T}.
+   *
+   * <p>This method attempts to infer the accumulator coder automatically.
+   *
+   * @see #value(Coder)
+   */
   public static <T> StateSpec<ValueState<T>> value() {
     return new ValueStateSpec<>(null);
   }
 
-  /** Create a simple state spec for values of type {@code T}. */
+  /**
+   * Identical to {@link #value()}, but with a coder explicitly supplied.
+   *
+   * <p>If automatic coder inference fails, use this method.
+   */
   public static <T> StateSpec<ValueState<T>> value(Coder<T> valueCoder) {
     checkArgument(valueCoder != null, "valueCoder should not be null. Consider value() instead");
     return new ValueStateSpec<>(valueCoder);
   }
 
   /**
-   * Create a state spec for values that use a {@link CombineFn} to automatically merge multiple
-   * {@code InputT}s into a single {@code OutputT}.
+   * Create a {@link StateSpec} for a {@link CombiningState} which uses a {@link CombineFn} to
+   * automatically merge multiple values of type {@code InputT} into a single resulting {@code
+   * OutputT}.
+   *
+   * <p>This method attempts to infer the accumulator coder automatically.
+   *
+   * @see #combining(Coder, CombineFn)
    */
   public static <InputT, AccumT, OutputT>
-  StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
-      CombineFn<InputT, AccumT, OutputT> combineFn) {
+      StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
+          CombineFn<InputT, AccumT, OutputT> combineFn) {
     return new CombiningStateSpec<InputT, AccumT, OutputT>(null, combineFn);
   }
 
   /**
-   * Create a state spec for values that use a {@link CombineFnWithContext} to automatically merge
-   * multiple {@code InputT}s into a single {@code OutputT}.
+   * Create a {@link StateSpec} for a {@link CombiningState} which uses a {@link
+   * CombineFnWithContext} to automatically merge multiple values of type {@code InputT} into a
+   * single resulting {@code OutputT}.
+   *
+   * <p>This method attempts to infer the accumulator coder automatically.
+   *
+   * @see #combining(Coder, CombineFnWithContext)
    */
   public static <InputT, AccumT, OutputT>
       StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
@@ -73,8 +91,9 @@ StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
   }
 
   /**
-   * Create a state spec for values that use a {@link CombineFn} to automatically merge multiple
-   * {@code InputT}s into a single {@code OutputT}.
+   * Identical to {@link #combining(CombineFn)}, but with an accumulator coder explicitly supplied.
+   *
+   * <p>If automatic coder inference fails, use this method.
    */
   public static <InputT, AccumT, OutputT>
   StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
@@ -86,8 +105,10 @@ StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
   }
 
   /**
-   * Create a state spec for values that use a {@link CombineFnWithContext} to automatically merge
-   * multiple {@code InputT}s into a single {@code OutputT}.
+   * Identical to {@link #combining(CombineFnWithContext)}, but with an accumulator coder explicitly
+   * supplied.
+   *
+   * <p>If automatic coder inference fails, use this method.
    */
   public static <InputT, AccumT, OutputT>
       StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
@@ -96,43 +117,62 @@ StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
   }
 
   /**
-   * Create a state spec that is optimized for adding values frequently, and occasionally retrieving
-   * all the values that have been added.
+   * Create a {@link StateSpec} for a {@link BagState}, optimized for adding values frequently
+   * and occasionally retrieving all the values that have been added.
+   *
+   * <p>This method attempts to infer the element coder automatically.
+   *
+   * @see #bag(Coder)
    */
   public static <T> StateSpec<BagState<T>> bag() {
     return bag(null);
   }
 
   /**
-   * Create a state spec that is optimized for adding values frequently, and occasionally retrieving
-   * all the values that have been added.
+   * Identical to {@link #bag()}, but with an element coder explicitly supplied.
+   *
+   * <p>If automatic coder inference fails, use this method.
    */
   public static <T> StateSpec<BagState<T>> bag(Coder<T> elemCoder) {
     return new BagStateSpec<>(elemCoder);
   }
 
   /**
-   * Create a state spec that supporting for {@link java.util.Set} like access patterns.
+   * Create a {@link StateSpec} for a {@link SetState}, optimized for checking membership.
+   *
+   * <p>This method attempts to infer the element coder automatically.
+   *
+   * @see #set(Coder)
    */
   public static <T> StateSpec<SetState<T>> set() {
     return set(null);
   }
 
   /**
-   * Create a state spec that supporting for {@link java.util.Set} like access patterns.
+   * Identical to {@link #set()}, but with an element coder explicitly supplied.
+   *
+   * <p>If automatic coder inference fails, use this method.
    */
   public static <T> StateSpec<SetState<T>> set(Coder<T> elemCoder) {
     return new SetStateSpec<>(elemCoder);
   }
 
   /**
-   * Create a state spec that supporting for {@link java.util.Map} like access patterns.
+   * Create a {@link StateSpec} for a {@link SetState}, optimized for key lookups and writes.
+   *
+   * <p>This method attempts to infer the key and value coders automatically.
+   *
+   * @see #map(Coder, Coder)
    */
   public static <K, V> StateSpec<MapState<K, V>> map() {
     return new MapStateSpec<>(null, null);
   }
 
-  /** Create a state spec that supporting for {@link java.util.Map} like access patterns. */
+  /**
+   * Identical to {@link #map()}, but with key and value coders explicitly supplied.
+   *
+   * <p>If automatic coder inference fails, use this method.
+   */
   public static <K, V> StateSpec<MapState<K, V>> map(Coder<K> keyCoder, Coder<V> valueCoder) {
     return new MapStateSpec<>(keyCoder, valueCoder);
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
index 0526453067b8..1778047a0558 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimeDomain.java
@@ -18,6 +18,7 @@
 package org.apache.beam.sdk.state;
 
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Internal;
 
 /**
  * {@link TimeDomain} specifies whether an operation is based on timestamps of elements or current
@@ -38,8 +39,11 @@ public enum TimeDomain {
   PROCESSING_TIME,
 
   /**
-   * Same as the {@link #PROCESSING_TIME} domain, except it won't fire a timer set for time <i>t</i>
-   * until all timers from earlier stages set for a time earlier than <i>t</i> have fired.
+   * <b>For internal use only; no backwards compatibility guarantees.</b>
+   *
+   * <p>Same as the {@link #PROCESSING_TIME} domain, except it won't fire a timer set for time
+   * <i>t</i> until all timers from earlier stages set for a time earlier than <i>t</i> have fired.
    */
+  @Internal
   SYNCHRONIZED_PROCESSING_TIME
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
index 9458906721b5..6f98ff82ee2e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
@@ -48,8 +48,8 @@ public interface Timer {
    * was already set, resets it to the new requested time.
    *
    * <p>For {@link TimeDomain#PROCESSING_TIME}, the behavior is be unpredictable, since processing
-   * time timers are ignored after a window has expired. Instead, it is recommended to use
-   * {@link #setRelative()}.
+   * time timers are ignored after a window has expired. Instead, it is recommended to use {@link
+   * #setRelative()}.
    */
   void set(Instant absoluteTime);
 
@@ -65,14 +65,12 @@ public interface Timer {
    */
   void setRelative();
 
-  /**
-   * Set the align offset.
-   */
+  /** Offsets the target timestamp used by {@link #setRelative()} by the given duration. */
   Timer offset(Duration offset);
 
   /**
-   * Aligns a timestamp to the next boundary of {@code period}.
+   * Aligns the target timestamp used by {@link #setRelative()} to the next boundary of {@code
+   * period}.
    */
   Timer align(Duration period);
-
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
index df4242872905..8b061eb26526 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/TimerSpecs.java
@@ -21,9 +21,7 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
-/**
- * Static methods for working with {@link TimerSpec}.
- */
+/** Static methods for working with {@link TimerSpec TimerSpecs}. */
 @Experimental(Kind.TIMERS)
 public class TimerSpecs {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ValueState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ValueState.java
index ca97db2fae4b..0562c89dde44 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ValueState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/ValueState.java
@@ -21,15 +21,13 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
- * State holding a single value.
+ * A {@link ReadableState} cell containing a single value.
  *
- * @param <T> The type of values being stored.
+ * @param <T> The type of value being stored.
  */
 @Experimental(Kind.STATE)
 public interface ValueState<T> extends ReadableState<T>, State {
-  /**
-   * Set the value of the buffer.
-   */
+  /** Set the value. */
   void write(T input);
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
index 38e2cbc27b64..6d4183da101f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/WatermarkHoldState.java
@@ -24,8 +24,8 @@
 /**
  * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
  *
- * <p>A {@link State} accepting and aggregating output timestamps, which determines the time to
- * which the output watermark must be held.
+ * <p>A {@link State} accepting and aggregating timestamps according to a {@link TimestampCombiner},
+ * holding the output watermark to the resulting timestamp.
  */
 @Internal
 public interface WatermarkHoldState extends GroupingState<Instant, Instant> {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java
index eb0a7acbcac6..2e0c1f795b54 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java
@@ -27,9 +27,7 @@
 import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger;
 import org.joda.time.Instant;
 
-/**
- * Create a {@link Trigger} that fires and finishes once after all of its sub-triggers have fired.
- */
+/** A composite {@link Trigger} that fires when all of its sub-triggers are ready. */
 @Experimental(Experimental.Kind.TRIGGER)
 public class AfterAll extends OnceTrigger {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java
index f0beb0a6d82b..f01801907afd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java
@@ -27,8 +27,7 @@
 import org.joda.time.Instant;
 
 /**
- * Create a composite {@link Trigger} that fires once after at least one of its sub-triggers have
- * fired.
+ * A composite {@link Trigger} that fires once after at least one of its sub-triggers have fired.
  */
 @Experimental(Experimental.Kind.TRIGGER)
 public class AfterFirst extends OnceTrigger {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java
index eade95d1350a..5914beb80d18 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterPane.java
@@ -24,7 +24,8 @@
 import org.joda.time.Instant;
 
 /**
- * {@link Trigger}s that fire based on properties of the elements in the current pane.
+ * A {@link Trigger} that fires at some point after a specified number of input elements have
+ * arrived.
  */
 @Experimental(Experimental.Kind.TRIGGER)
 public class AfterPane extends OnceTrigger {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java
index cc7ec137b87f..00e62befc126 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java
@@ -30,8 +30,8 @@
 import org.joda.time.format.PeriodFormatter;
 
 /**
- * {@code AfterProcessingTime} triggers fire based on the current processing time. They operate in
- * the real-time domain.
+ * A {@link Trigger} trigger that fires at a specified point in processing time, relative to when
+ * input first arrives.
  */
 @Experimental(Experimental.Kind.TRIGGER)
 public class AfterProcessingTime extends OnceTrigger {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java
index 78f373540fd2..ac00877139d4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/DefaultTrigger.java
@@ -22,11 +22,13 @@
 import org.joda.time.Instant;
 
 /**
- * A trigger that is equivalent to {@code Repeatedly.forever(AfterWatermark.pastEndOfWindow())}.
- * See {@link Repeatedly#forever} and {@link AfterWatermark#pastEndOfWindow} for more details.
+ * A trigger that is equivalent to {@code Repeatedly.forever(AfterWatermark.pastEndOfWindow())}. See
+ * {@link Repeatedly#forever} and {@link AfterWatermark#pastEndOfWindow} for more details.
+ *
+ * <p>This is a distinguished class to make it easy for runners to optimize for this common case.
  */
 @Experimental(Experimental.Kind.TRIGGER)
-public class DefaultTrigger extends Trigger{
+public class DefaultTrigger extends Trigger {
 
   private DefaultTrigger() {
     super();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
index 400be1fe7095..bc3438b9ec55 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
@@ -23,7 +23,10 @@
 import org.joda.time.Instant;
 
 /**
- * Default {@link WindowFn} that assigns all data to the same window.
+ * A {@link WindowFn} that assigns all data to the same window.
+ *
+ * <p>This is the {@link WindowFn} used for data coming from a source, before a
+ * {@link Window} transform has been applied.
  */
 public class GlobalWindows extends NonMergingWindowFn<Object, GlobalWindow> {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
index 92041fc54c73..49535a06bd56 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
@@ -36,7 +36,7 @@ public InvalidWindows(String cause, WindowFn<?, W> originalWindowFn) {
   }
 
   /**
-   * Returns the reason that this {@code WindowFn} is invalid.
+   * Returns the reason that this {@link WindowFn} is invalid.
    */
   public String getCause() {
     return cause;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java
index 8314dd7fae7e..0a6802145901 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/MergeOverlappingIntervalWindows.java
@@ -21,10 +21,14 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import org.apache.beam.sdk.annotations.Internal;
 
 /**
- * A utility function for merging overlapping {@link IntervalWindow}s.
+ * <b>For internal use only; no backwards compatibility guarantees.</b>
+ *
+ * <p>A utility function for merging overlapping {@link IntervalWindow IntervalWindows}.
  */
+@Internal
 public class MergeOverlappingIntervalWindows {
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java
index 6dfeea72067f..b9acd246d678 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Never.java
@@ -23,7 +23,7 @@
 import org.joda.time.Instant;
 
 /**
- * A trigger which never fires.
+ * A {@link Trigger} which never fires.
  *
  * <p>Using this trigger will only produce output when the watermark passes the end of the
  * {@link BoundedWindow window} plus the {@link Window#withAllowedLateness allowed
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/NonMergingWindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/NonMergingWindowFn.java
index d304a68116eb..5cedc6b90cd5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/NonMergingWindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/NonMergingWindowFn.java
@@ -18,7 +18,7 @@
 package org.apache.beam.sdk.transforms.windowing;
 
 /**
- * Abstract base class for {@link WindowFn}s that do not merge windows.
+ * Abstract base class for {@link WindowFn WindowFns} that do not merge windows.
  *
  * @param <T> type of elements being windowed
  * @param <W> {@link BoundedWindow} subclass used to represent the windows used by this
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java
index ad0de47e0bbb..7eefb45eaaf8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java
@@ -23,7 +23,10 @@
 import org.joda.time.Instant;
 
 /**
- * Executes the {@code actual} trigger until it finishes or until the {@code until} trigger fires.
+ * A {@link Trigger} that executes according to its main trigger until its "finally" trigger fires.
+ *
+ * <p>Uniquely among triggers, the "finally" trigger's predicate is applied to all input seen so
+ * far, not input since the last firing.
  */
 public class OrFinallyTrigger extends Trigger {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java
index 78b79c74fbab..43951a368bcf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Repeatedly.java
@@ -22,7 +22,7 @@
 import org.joda.time.Instant;
 
 /**
- * Repeat a trigger, either until some condition is met or forever.
+ * A {@link Trigger} that fires according to its subtrigger forever.
  *
  * <p>For example, to fire after the end of the window, and every time late data arrives:
  * <pre> {@code
@@ -31,6 +31,8 @@
  *
  * <p>{@code Repeatedly.forever(someTrigger)} behaves like an infinite
  * {@code AfterEach.inOrder(someTrigger, someTrigger, someTrigger, ...)}.
+ *
+ * <p>You can use {@link #orFinally(OnceTrigger)} to let another trigger interrupt the repetition.
  */
 public class Repeatedly extends Trigger {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
index 5cc7c65b7da5..739072852a2a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
@@ -25,16 +25,17 @@
 import org.joda.time.Duration;
 
 /**
- * A {@link WindowFn} windowing values into sessions separated by {@link #gapDuration}-long
- * periods with no elements.
+ * A {@link WindowFn} that windows values into sessions separated by periods with no input for at
+ * least the duration specified by {@link #getGapDuration()}.
  *
- * <p>For example, in order to window data into session with at least 10 minute
- * gaps in between them:
- * <pre> {@code
+ * <p>For example, in order to window data into session with at least 10 minute gaps in between
+ * them:
+ *
+ * <pre>{@code
  * PCollection<Integer> pc = ...;
  * PCollection<Integer> windowed_pc = pc.apply(
  *   Window.<Integer>into(Sessions.withGapDuration(Duration.standardMinutes(10))));
- * } </pre>
+ * }</pre>
  */
 public class Sessions extends WindowFn<Object, IntervalWindow> {
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java
index 39fe8a9171d7..6cdcc3b445f4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java
@@ -24,10 +24,15 @@
 import java.util.Arrays;
 import java.util.Collections;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.GroupByKey;
 import org.joda.time.Instant;
 
 /**
  * Policies for combining timestamps that occur within a window.
+ *
+ * <p>In particular, these govern the timestamp on the output of a grouping transform
+ * such as {@link GroupByKey} or {@link Combine}.
  */
 @Experimental(Experimental.Kind.OUTPUT_TIME)
 public enum TimestampCombiner {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index 27771912f2dd..dc4863b06b0c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -37,7 +37,7 @@
 import org.joda.time.Duration;
 
 /**
- * {@code Window} logically divides up or groups the elements of a
+ * {@link Window} logically divides up or groups the elements of a
  * {@link PCollection} into finite windows according to a {@link WindowFn}.
  * The output of {@code Window} contains the same elements as input, but they
  * have been logically assigned to windows. The next
@@ -50,16 +50,16 @@
  *
  * <h2>Windowing</h2>
  *
- * <p>Windowing a {@code PCollection} divides the elements into windows based
+ * <p>Windowing a {@link PCollection} divides the elements into windows based
  * on the associated event time for each element. This is especially useful
- * for {@code PCollection}s with unbounded size, since it allows operating on
- * a sub-group of the elements placed into a related window. For {@code PCollection}s
+ * for {@link PCollection PCollections} with unbounded size, since it allows operating on
+ * a sub-group of the elements placed into a related window. For {@link PCollection PCollections}
  * with a bounded size (aka. conventional batch mode), by default, all data is
- * implicitly in a single window, unless {@code Window} is applied.
+ * implicitly in a single window, unless {@link Window} is applied.
  *
  * <p>For example, a simple form of windowing divides up the data into
  * fixed-width time intervals, using {@link FixedWindows}.
- * The following example demonstrates how to use {@code Window} in a pipeline
+ * The following example demonstrates how to use {@link Window} in a pipeline
  * that counts the number of occurrences of strings each minute:
  *
  * <pre>{@code
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
index 5ebbb41b5956..52ebd61d5dec 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
@@ -31,21 +31,21 @@
  * The argument to the {@link Window} transform used to assign elements into
  * windows and to determine how windows are merged.  See {@link Window} for more
  * information on how {@code WindowFn}s are used and for a library of
- * predefined {@code WindowFn}s.
+ * predefined {@link WindowFn WindowFns}.
  *
  * <p>Users will generally want to use the predefined
- * {@code WindowFn}s, but it is also possible to create new
+ * {@link WindowFn WindowFns}, but it is also possible to create new
  * subclasses.
  *
- * <p>To create a custom {@code WindowFn}, inherit from this class and override all required
+ * <p>To create a custom {@link WindowFn}, inherit from this class and override all required
  * methods.  If no merging is required, inherit from {@link NonMergingWindowFn}
  * instead.  If no merging is required and each element is assigned to a single window, inherit from
- * {@code PartitioningWindowFn}.  Inheriting from the most specific subclass will enable more
+ * {@link PartitioningWindowFn}.  Inheriting from the most specific subclass will enable more
  * optimizations in the runner.
  *
  * @param <T> type of elements being windowed
  * @param <W> {@link BoundedWindow} subclass used to represent the
- *            windows used by this {@code WindowFn}
+ *            windows used by this {@link WindowFn}
  */
 public abstract class WindowFn<T, W extends BoundedWindow>
     implements Serializable, HasDisplayData {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowMappingFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowMappingFn.java
index c144aba3a381..5da9db17da30 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowMappingFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowMappingFn.java
@@ -19,15 +19,19 @@
 package org.apache.beam.sdk.transforms.windowing;
 
 import java.io.Serializable;
-import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
+import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Duration;
 
 /**
- * A function that takes the windows of elements in a main input and maps them to the appropriate
- * window in a {@link PCollectionView} consumed as a
- * {@link MultiOutput#withSideInputs(PCollectionView[]) side input}.
+ * <b>Experimental! This will be ready for users eventually, but should be considered internal for
+ * now. No backwards compatibility guarantees.</b>
+ *
+ * <p>A function that takes the windows of elements in a main input and maps them to the appropriate
+ * window in a {@link PCollectionView} consumed as a {@link
+ * org.apache.beam.sdk.transforms.ParDo.MultiOutput#withSideInputs(PCollectionView[]) side input}.
  */
+@Experimental
 public abstract class WindowMappingFn<TargetWindowT extends BoundedWindow> implements Serializable {
   private final Duration maximumLookback;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PBegin.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PBegin.java
index 04d1bdb59b4b..bf26e1b84f6f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PBegin.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PBegin.java
@@ -21,37 +21,35 @@
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.TextIO.Read;
-import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
  * {@link PBegin} is the "input" to a root {@link PTransform}, such as {@link Read Read} or
- * {@link Create}.
+ * {@link org.apache.beam.sdk.transforms.Create}.
  *
- * <p>Typically created by calling {@link Pipeline#begin} on a Pipeline.
+ * <p>Typically elided by simply calling {@link Pipeline#apply(String, PTransform)} or {@link
+ * Pipeline#apply(PTransform)}, but one can be explicitly created by calling {@link Pipeline#begin}
+ * on a Pipeline.
  */
 public class PBegin implements PInput {
-  /**
-   * Returns a {@link PBegin} in the given {@link Pipeline}.
-   */
+  /** Returns a {@link PBegin} in the given {@link Pipeline}. */
   public static PBegin in(Pipeline pipeline) {
     return new PBegin(pipeline);
   }
 
   /**
-   * Like {@link #apply(String, PTransform)} but defaulting to the name
-   * of the {@link PTransform}.
+   * Like {@link #apply(String, PTransform)} but defaulting to the name of the {@link PTransform}.
    */
-  public <OutputT extends POutput> OutputT apply(
-      PTransform<? super PBegin, OutputT> t) {
+  public <OutputT extends POutput> OutputT apply(PTransform<? super PBegin, OutputT> t) {
     return Pipeline.applyTransform(this, t);
   }
 
   /**
-   * Applies the given {@link PTransform} to this input {@link PBegin},
-   * using {@code name} to identify this specific application of the transform.
-   * This name is used in various places, including the monitoring UI, logging,
-   * and to stably identify this application node in the job graph.
+   * Applies the given {@link PTransform} to this {@link PBegin}, using {@code name} to identify
+   * this specific application of the transform.
+   *
+   * <p>This name is used in various places, including the monitoring UI, logging, and to stably
+   * identify this application node in the job graph.
    */
   public <OutputT extends POutput> OutputT apply(
       String name, PTransform<? super PBegin, OutputT> t) {

From fd93ff658b2da36c4f7c7c5289484fb6493b609b Mon Sep 17 00:00:00 2001
From: Maria Garcia Herrero <mariagh@google.com>
Date: Tue, 9 May 2017 13:58:19 -0700
Subject: [PATCH 152/387] Remove templates from wordcount example

---
 sdks/python/apache_beam/examples/wordcount.py | 31 +++++++++----------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py
index 8c6a485bc7a7..e7e542a74219 100644
--- a/sdks/python/apache_beam/examples/wordcount.py
+++ b/sdks/python/apache_beam/examples/wordcount.py
@@ -19,6 +19,7 @@
 
 from __future__ import absolute_import
 
+import argparse
 import logging
 import re
 
@@ -66,29 +67,25 @@ def process(self, element):
 
 def run(argv=None):
   """Main entry point; defines and runs the wordcount pipeline."""
-  class WordcountOptions(PipelineOptions):
-    @classmethod
-    def _add_argparse_args(cls, parser):
-      parser.add_value_provider_argument(
-          '--input',
-          dest='input',
-          default='gs://dataflow-samples/shakespeare/kinglear.txt',
-          help='Input file to process.')
-      parser.add_value_provider_argument(
-          '--output',
-          dest='output',
-          required=True,
-          help='Output file to write results to.')
-  pipeline_options = PipelineOptions(argv)
-  wordcount_options = pipeline_options.view_as(WordcountOptions)
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--input',
+                      dest='input',
+                      default='gs://dataflow-samples/shakespeare/kinglear.txt',
+                      help='Input file to process.')
+  parser.add_argument('--output',
+                      dest='output',
+                      required=True,
+                      help='Output file to write results to.')
+  known_args, pipeline_args = parser.parse_known_args(argv)
 
   # We use the save_main_session option because one or more DoFn's in this
   # workflow rely on global context (e.g., a module imported at module level).
+  pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
   p = beam.Pipeline(options=pipeline_options)
 
   # Read the text file[pattern] into a PCollection.
-  lines = p | 'read' >> ReadFromText(wordcount_options.input)
+  lines = p | 'read' >> ReadFromText(known_args.input)
 
   # Count the occurrences of each word.
   counts = (lines
@@ -103,7 +100,7 @@ def _add_argparse_args(cls, parser):
 
   # Write the output using a "Write" transform that has side effects.
   # pylint: disable=expression-not-assigned
-  output | 'write' >> WriteToText(wordcount_options.output)
+  output | 'write' >> WriteToText(known_args.output)
 
   # Actually run the pipeline (all operations above are deferred).
   result = p.run()

From 1f631eb9f02c32f1a78712dcfb5c1bba9c9d4e93 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Tue, 9 May 2017 15:22:11 -0700
Subject: [PATCH 153/387] Remove hadoop io readme

---
 .../apache/beam/examples/complete/README.md   |   2 +-
 .../apache/beam/examples/cookbook/README.md   |   2 +-
 sdks/java/io/hadoop/README.md                 | 167 ------------------
 3 files changed, 2 insertions(+), 169 deletions(-)
 delete mode 100644 sdks/java/io/hadoop/README.md

diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/README.md b/examples/java/src/main/java/org/apache/beam/examples/complete/README.md
index 2b416c65f06f..3f4842a5c7a7 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/README.md
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/README.md
@@ -58,5 +58,5 @@ This directory contains end-to-end example pipelines that perform complex data p
   </ul>
 
 See the [documentation](http://beam.apache.org/get-started/quickstart/) and the [Examples
-README](../../../../../../../../../README.md) for
+README](../../../../../../../../README.md) for
 information about how to run these examples.
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/README.md b/examples/java/src/main/java/org/apache/beam/examples/cookbook/README.md
index b167cd7a937b..a40580817c0b 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/README.md
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/README.md
@@ -67,5 +67,5 @@ larger Apache Beam pipeline. They include:
   </ul>
 
 See the [documentation](http://beam.apache.org/get-started/quickstart/) and the [Examples
-README](../../../../../../../../../README.md) for
+README](../../../../../../../../README.md) for
 information about how to run these examples.
diff --git a/sdks/java/io/hadoop/README.md b/sdks/java/io/hadoop/README.md
deleted file mode 100644
index d91f019cc25f..000000000000
--- a/sdks/java/io/hadoop/README.md
+++ /dev/null
@@ -1,167 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-# Hadoop InputFormat IO
-
-A HadoopInputFormatIO is a Transform for reading data from any source which
-implements Hadoop InputFormat. For example- Cassandra, Elasticsearch, HBase, Redis, Postgres, etc.
-
-HadoopInputFormatIO has to make several performance trade-offs in connecting to InputFormat, so if there is another Beam IO Transform specifically for connecting to your data source of choice, we would recommend using that one, but this IO Transform allows you to connect to many data sources that do not yet have a Beam IO Transform.
-
-You will need to pass a Hadoop Configuration with parameters specifying how the read will occur. Many properties of the Configuration are optional, and some are required for certain InputFormat classes, but the following properties must be set for all InputFormats:
-
-mapreduce.job.inputformat.class: The InputFormat class used to connect to your data source of choice.
-key.class: The key class returned by the InputFormat in 'mapreduce.job.inputformat.class'.
-value.class: The value class returned by the InputFormat in 'mapreduce.job.inputformat.class'.
-
-For example:
-```java
-Configuration myHadoopConfiguration = new Configuration(false);
-// Set Hadoop InputFormat, key and value class in configuration
-myHadoopConfiguration.setClass("mapreduce.job.inputformat.class", InputFormatClass,
-  InputFormat.class);
-myHadoopConfiguration.setClass("key.class", InputFormatKeyClass, Object.class);
-myHadoopConfiguration.setClass("value.class", InputFormatValueClass, Object.class);
-```
-
-You will need to check to see if the key and value classes output by the InputFormat have a Beam Coder available. If not, You can use withKeyTranslation/withValueTranslation to specify a method transforming instances of those classes into another class that is supported by a Beam Coder. These settings are optional and you don't need to specify translation for both key and value.
-
-For example:
-```java
-SimpleFunction<InputFormatKeyClass, MyKeyClass> myOutputKeyType =
-new SimpleFunction<InputFormatKeyClass, MyKeyClass>() {
-  public MyKeyClass apply(InputFormatKeyClass input) {
-  // ...logic to transform InputFormatKeyClass to MyKeyClass
-  }
-};
-SimpleFunction<InputFormatValueClass, MyValueClass> myOutputValueType =
-new SimpleFunction<InputFormatValueClass, MyValueClass>() {
-  public MyValueClass apply(InputFormatValueClass input) {
-  // ...logic to transform InputFormatValueClass to MyValueClass
-  }
-};
-```
-
-### Reading using Hadoop InputFormat IO
-Pipeline p = ...; // Create pipeline.
-// Read data only with Hadoop configuration.
-
-```java
-p.apply("read",
-  HadoopInputFormatIO.<InputFormatKeyClass, InputFormatKeyClass>read()
-  .withConfiguration(myHadoopConfiguration);
-```
-
-// Read data with configuration and key translation (Example scenario: Beam Coder is not
-available for key class hence key translation is required.).
-
-```java
-p.apply("read",
-  HadoopInputFormatIO.<MyKeyClass, InputFormatKeyClass>read()
-  .withConfiguration(myHadoopConfiguration)
-  .withKeyTranslation(myOutputKeyType);
-```
-
-// Read data with configuration and value translation (Example scenario: Beam Coder is not
-available for value class hence value translation is required.).
-
-```java
-p.apply("read",
-  HadoopInputFormatIO.<InputFormatKeyClass, MyValueClass>read()
-  .withConfiguration(myHadoopConfiguration)
-  .withValueTranslation(myOutputValueType);
-```
-
-// Read data with configuration, value translation and key translation (Example scenario: Beam Coders are not available for both key class and value class of InputFormat hence key and value translation is required.).
-
-```java
-p.apply("read",
-  HadoopInputFormatIO.<MyKeyClass, MyValueClass>read()
-  .withConfiguration(myHadoopConfiguration)
-  .withKeyTranslation(myOutputKeyType)
-  .withValueTranslation(myOutputValueType);
-```
-
-# Examples for specific InputFormats
-
-### Cassandra - CqlInputFormat
-
-To read data from Cassandra, org.apache.cassandra.hadoop.cql3.CqlInputFormat
-CqlInputFormat can be used which needs following properties to be set.
-
-Create Cassandra Hadoop configuration as follows:
-
-```java
-Configuration cassandraConf = new Configuration();
-cassandraConf.set("cassandra.input.thrift.port", "9160");
-cassandraConf.set("cassandra.input.thrift.address", CassandraHostIp);
-cassandraConf.set("cassandra.input.partitioner.class", "Murmur3Partitioner");
-cassandraConf.set("cassandra.input.keyspace", "myKeySpace");
-cassandraConf.set("cassandra.input.columnfamily", "myColumnFamily");
-cassandraConf.setClass("key.class", java.lang.Long Long.class, Object.class);
-cassandraConf.setClass("value.class", com.datastax.driver.core.Row Row.class, Object.class);
-cassandraConf.setClass("mapreduce.job.inputformat.class", org.apache.cassandra.hadoop.cql3.CqlInputFormat CqlInputFormat.class, InputFormat.class);
-```
-
-Call Read transform as follows:
-
-```java
-PCollection<KV<Long, String>> cassandraData =
-  p.apply("read",
-  HadoopInputFormatIO.<Long, String>read()
-  .withConfiguration(cassandraConf)
-  .withValueTranslation(cassandraOutputValueType);
-```
-
-The CqlInputFormat key class is java.lang.Long Long, which has a Beam Coder. The CqlInputFormat value class is com.datastax.driver.core.Row Row, which does not have a Beam Coder. Rather than write a new coder, you can provide your own translation method as follows:
-
-```java
-SimpleFunction<Row, String> cassandraOutputValueType = SimpleFunction<Row, String>()
-{
-  public String apply(Row row) {
-    return row.getString('myColName');
-  }
-};
-```
- 
-### Elasticsearch - EsInputFormat
- 
-To read data from Elasticsearch, EsInputFormat can be used which needs following properties to be set.
- 
-Create ElasticSearch Hadoop configuration as follows:
-
-```java
-Configuration elasticSearchConf = new Configuration();
-elasticSearchConf.set("es.nodes", ElasticsearchHostIp);
-elasticSearchConf.set("es.port", "9200");
-elasticSearchConf.set("es.resource", "ElasticIndexName/ElasticTypeName");
-elasticSearchConf.setClass("key.class", org.apache.hadoop.io.Text Text.class, Object.class);
-elasticSearchConf.setClass("value.class", org.elasticsearch.hadoop.mr.LinkedMapWritable LinkedMapWritable.class, Object.class);
-elasticSearchConf.setClass("mapreduce.job.inputformat.class", org.elasticsearch.hadoop.mr.EsInputFormat EsInputFormat.class, InputFormat.class);
-```
-
-Call Read transform as follows:
-
-```java
-PCollection<KV<Text, LinkedMapWritable>> elasticData = p.apply("read",
-  HadoopInputFormatIO.<Text, LinkedMapWritable>read().withConfiguration(elasticSearchConf));
-```
-
-The org.elasticsearch.hadoop.mr.EsInputFormat EsInputFormat key class is
-org.apache.hadoop.io.Text Text and value class is org.elasticsearch.hadoop.mr.LinkedMapWritable LinkedMapWritable. Both key and value classes have Beam Coders.
\ No newline at end of file

From ded60a724cbceadb873b0edcd82ae6a3a7835803 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 14:11:47 -0700
Subject: [PATCH 154/387] Shade JSR305 in the DirectRunner

None of the APIs exposed by the DirectRunner are nullable.
---
 runners/direct-java/pom.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 19ee81a02c08..c115906efb7e 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -95,6 +95,7 @@
                   <include>org.apache.beam:beam-runners-core-construction-java</include>
                   <include>org.apache.beam:beam-runners-core-java</include>
                   <include>org.apache.beam:beam-sdks-common-runner-api</include>
+                  <include>com.google.code.findbugs:jsr305</include>
                 </includes>
               </artifactSet>
               <filters>
@@ -138,6 +139,12 @@
                     org.apache.beam.runners.direct.repackaged.com.google.thirdparty
                   </shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>javax.annotation</pattern>
+                  <shadedPattern>
+                    org.apache.beam.runners.direct.repackaged.javax.annotation
+                  </shadedPattern>
+                </relocation>
               </relocations>
               <transformers>
                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>

From 566469eff3095ffd186a04cb46e5e4cbd7d05724 Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Tue, 9 May 2017 16:22:19 -0700
Subject: [PATCH 155/387] Update Apache Beam Python version to 2.1.0.dev

---
 sdks/python/apache_beam/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py
index 22d16839e567..ae92a235c53b 100644
--- a/sdks/python/apache_beam/version.py
+++ b/sdks/python/apache_beam/version.py
@@ -18,4 +18,4 @@
 """Apache Beam SDK version information and utilities."""
 
 
-__version__ = '0.7.0.dev'
+__version__ = '2.1.0.dev'

From 95d33c521788ce8046201a6baf22e46950560cf1 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 9 May 2017 12:44:19 -0700
Subject: [PATCH 156/387] Register TestSparkPipelineOptions only in src/test to
 avoid hard hamcrest dep

---
 .../runners/spark/SparkRunnerRegistrar.java   |  4 +--
 .../beam/runners/spark/TestSparkRunner.java   | 29 ++++++++-------
 .../spark/SparkRunnerRegistrarTest.java       |  2 +-
 .../TestSparkPipelineOptionsRegistrar.java    | 36 +++++++++++++++++++
 4 files changed, 52 insertions(+), 19 deletions(-)
 create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
index e2e5ceba0221..325c86dfa8c5 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java
@@ -54,9 +54,7 @@ public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
   public static class Options implements PipelineOptionsRegistrar {
     @Override
     public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
-      return ImmutableList.<Class<? extends PipelineOptions>>of(
-          SparkPipelineOptions.class,
-          TestSparkPipelineOptions.class);
+      return ImmutableList.<Class<? extends PipelineOptions>>of(SparkPipelineOptions.class);
     }
   }
 }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 6d10b7588a42..eccee574afaa 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -83,26 +83,25 @@
 public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestSparkRunner.class);
-  private final TestSparkPipelineOptions testSparkPipelineOptions;
-
+  private final PipelineOptions options;
   private SparkRunner delegate;
-  private boolean isForceStreaming;
 
-  private TestSparkRunner(TestSparkPipelineOptions options) {
+  private TestSparkRunner(PipelineOptions options) {
     this.delegate = SparkRunner.fromOptions(options);
-    this.isForceStreaming = options.isForceStreaming();
-    this.testSparkPipelineOptions = options;
+    this.options = options;
   }
 
   public static TestSparkRunner fromOptions(PipelineOptions options) {
-    // Default options suffice to set it up as a test runner
-    TestSparkPipelineOptions sparkOptions =
-        PipelineOptionsValidator.validate(TestSparkPipelineOptions.class, options);
-    return new TestSparkRunner(sparkOptions);
+    return new TestSparkRunner(options);
   }
 
   @Override
   public SparkPipelineResult run(Pipeline pipeline) {
+    // Default options suffice to set it up as a test runner
+    TestSparkPipelineOptions testSparkOptions =
+        PipelineOptionsValidator.validate(TestSparkPipelineOptions.class, options);
+
+    boolean isForceStreaming = testSparkOptions.isForceStreaming();
     // if the pipeline forces execution as a streaming pipeline,
     // and the source is an adapted unbounded source (as bounded),
     // read it as unbounded source via UnboundedReadFromBoundedSource.
@@ -116,13 +115,13 @@ public SparkPipelineResult run(Pipeline pipeline) {
     MetricsAccumulator.clear();
     GlobalWatermarkHolder.clear();
 
-    LOG.info("About to run test pipeline " + testSparkPipelineOptions.getJobName());
+    LOG.info("About to run test pipeline " + options.getJobName());
 
     // if the pipeline was executed in streaming mode, validate aggregators.
     if (isForceStreaming) {
       try {
         result = delegate.run(pipeline);
-        awaitWatermarksOrTimeout(testSparkPipelineOptions, result);
+        awaitWatermarksOrTimeout(testSparkOptions, result);
         result.stop();
         PipelineResult.State finishState = result.getState();
         // assert finish state.
@@ -133,7 +132,7 @@ public SparkPipelineResult run(Pipeline pipeline) {
       } finally {
         try {
           // cleanup checkpoint dir.
-          FileUtils.deleteDirectory(new File(testSparkPipelineOptions.getCheckpointDir()));
+          FileUtils.deleteDirectory(new File(testSparkOptions.getCheckpointDir()));
         } catch (IOException e) {
           throw new RuntimeException("Failed to clear checkpoint tmp dir.", e);
         }
@@ -150,8 +149,8 @@ public SparkPipelineResult run(Pipeline pipeline) {
           finishState,
           is(PipelineResult.State.DONE));
       // assert via matchers.
-      assertThat(result, testSparkPipelineOptions.getOnCreateMatcher());
-      assertThat(result, testSparkPipelineOptions.getOnSuccessMatcher());
+      assertThat(result, testSparkOptions.getOnCreateMatcher());
+      assertThat(result, testSparkOptions.getOnSuccessMatcher());
     }
     return result;
   }
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
index 75899f9a395f..4e1fd7c5afee 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
@@ -38,7 +38,7 @@ public class SparkRunnerRegistrarTest {
   @Test
   public void testOptions() {
     assertEquals(
-        ImmutableList.of(SparkPipelineOptions.class, TestSparkPipelineOptions.class),
+        ImmutableList.of(SparkPipelineOptions.class),
         new SparkRunnerRegistrar.Options().getPipelineOptions());
   }
 
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java
new file mode 100644
index 000000000000..e71880b0e23d
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
+
+/**
+ * A registrar for {@link TestSparkPipelineOptions} to temporarily work around some complexities in
+ * {@link PipelineOptions} parsing.
+ */
+@AutoService(PipelineOptionsRegistrar.class)
+public final class TestSparkPipelineOptionsRegistrar implements PipelineOptionsRegistrar {
+  @Override
+  public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
+    return ImmutableList.<Class<? extends PipelineOptions>>of(TestSparkPipelineOptions.class);
+  }
+}

From 038febbd889cc98c5f6e579b31f84e19ff3981f5 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 9 May 2017 13:02:46 -0700
Subject: [PATCH 157/387] Add per-runner profile to Java 8 examples

---
 examples/java8/pom.xml | 83 +++++++++++++++++++++++++++---------------
 1 file changed, 54 insertions(+), 29 deletions(-)

diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 5a94beecfda2..3c9ed71ff89f 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -41,21 +41,11 @@
 
   <profiles>
     <!--
-      A default profile that includes optional dependencies
-      on all of our runners. This is aimed at making it very
-      easy for users to run examples with any runner without
-      any configuration. It is not needed or desirable when
-      testing the examples with a particular runner.
-
-      This profile can be disabled on the command line
-      by specifying -P !include-runners.
-
-      This profile cannot be lifted to examples-parent because
-      it would be automatically deactivated when the Java 8
-      profile were activated.
+      The direct runner is available by default.
+      You can also include it on the classpath explicitly with -P direct-runner
     -->
     <profile>
-      <id>include-runners</id>
+      <id>direct-runner</id>
       <activation>
         <activeByDefault>true</activeByDefault>
       </activation>
@@ -63,41 +53,72 @@
         <dependency>
           <groupId>org.apache.beam</groupId>
           <artifactId>beam-runners-direct-java</artifactId>
-          <version>${project.version}</version>
           <scope>runtime</scope>
-          <optional>true</optional>
         </dependency>
+      </dependencies>
+    </profile>
 
+    <!-- Include the Apache Apex runner with -P apex-runner -->
+    <profile>
+      <id>apex-runner</id>
+      <dependencies>
         <dependency>
           <groupId>org.apache.beam</groupId>
-          <artifactId>beam-runners-flink_2.10</artifactId>
-          <version>${project.version}</version>
+          <artifactId>beam-runners-apex</artifactId>
           <scope>runtime</scope>
-          <optional>true</optional>
         </dependency>
+        <!--
+          Apex depends on httpclient version 4.3.5, project has a transitive dependency to httpclient 4.0.1 from
+          google-http-client. Apex dependency version being specified explicitly so that it gets picked up. This
+          can be removed when the project no longer has a dependency on a different httpclient version.
+        -->
+        <dependency>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpclient</artifactId>
+          <version>4.3.5</version>
+          <scope>runtime</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>commons-codec</groupId>
+              <artifactId>commons-codec</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
+      </dependencies>
+    </profile>
 
+    <!-- Include the Apache Flink runner with -P flink-runner -->
+    <profile>
+      <id>flink-runner</id>
+      <dependencies>
         <dependency>
           <groupId>org.apache.beam</groupId>
-          <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
-          <version>${project.version}</version>
+          <artifactId>beam-runners-flink_2.10</artifactId>
           <scope>runtime</scope>
-          <optional>true</optional>
         </dependency>
+      </dependencies>
+    </profile>
 
+    <!-- Include the Apache Spark runner -P spark-runner -->
+    <profile>
+      <id>spark-runner</id>
+      <dependencies>
         <dependency>
           <groupId>org.apache.beam</groupId>
           <artifactId>beam-runners-spark</artifactId>
-          <version>${project.version}</version>
           <scope>runtime</scope>
-          <optional>true</optional>
         </dependency>
-
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-streaming_2.10</artifactId>
+          <version>${spark.version}</version>
+          <scope>runtime</scope>
+        </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-core_2.10</artifactId>
           <version>${spark.version}</version>
           <scope>runtime</scope>
-          <optional>true</optional>
           <exclusions>
             <exclusion>
               <groupId>org.slf4j</groupId>
@@ -105,13 +126,17 @@
             </exclusion>
           </exclusions>
         </dependency>
+      </dependencies>
+    </profile>
 
+    <!-- Include the Google Cloud Dataflow runner -P dataflow-runner -->
+    <profile>
+      <id>dataflow-runner</id>
+      <dependencies>
         <dependency>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-streaming_2.10</artifactId>
-          <version>${spark.version}</version>
+          <groupId>org.apache.beam</groupId>
+          <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
           <scope>runtime</scope>
-          <optional>true</optional>
         </dependency>
       </dependencies>
     </profile>

From dae9fc71e526a26966bd9296fa1f64e84361037e Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 9 May 2017 17:02:57 -0700
Subject: [PATCH 158/387] Remove trailing whitespace

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 1202c54d7fc7..52c056f59ac5 100644
--- a/README.md
+++ b/README.md
@@ -71,8 +71,8 @@ Have ideas for new Runners? See the [JIRA](https://issues.apache.org/jira/browse
 
 Please refer to the [Quickstart](http://beam.apache.org/get-started/quickstart/) available on our website.
 
-If you'd like to build and install the whole project from the source distribution, you may need some additional tools installed 
-in your system. In a Debian-based distribution: 
+If you'd like to build and install the whole project from the source distribution, you may need some additional tools installed
+in your system. In a Debian-based distribution:
 
 ```
 sudo apt-get install \

From 62f2173a1c03ba0433ab4941b7546f2d789b998f Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Tue, 9 May 2017 11:41:29 -0700
Subject: [PATCH 159/387] Shade dependencies in sdks/core

---
 sdks/java/core/pom.xml                        | 74 +++++++++++++++++++
 .../org/apache/beam/sdk/util/ApiSurface.java  | 10 ++-
 2 files changed, 83 insertions(+), 1 deletion(-)

diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 7ed6c8a3a553..ef1873d2dbf1 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -120,6 +120,80 @@
           </execution>
         </executions>
       </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>bundle-and-repackage</id>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <artifactSet>
+                <includes>
+                  <include>com.google.guava:guava</include>
+                  <include>com.google.protobuf:protobuf-java</include>
+                  <include>net.bytebuddy:byte-buddy</include>
+                  <include>org.apache.commons:*</include>
+                </includes>
+              </artifactSet>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <relocations>
+                <relocation>
+                  <pattern>com.google.common</pattern>
+                  <!--suppress MavenModelInspection -->
+                  <shadedPattern>
+                    org.apache.beam.sdk.repackaged.com.google.common
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.google.thirdparty</pattern>
+                  <!--suppress MavenModelInspection -->
+                  <shadedPattern>
+                    org.apache.beam.sdk.repackaged.com.google.thirdparty
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.google.protobuf</pattern>
+                  <!--suppress MavenModelInspection -->
+                  <shadedPattern>
+                    org.apache.beam.sdk.repackaged.com.google.protobuf
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>net.bytebuddy</pattern>
+                  <!--suppress MavenModelInspection -->
+                  <shadedPattern>
+                    org.apache.beam.sdk.repackaged.net.bytebuddy
+                  </shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.commons</pattern>
+                  <!--suppress MavenModelInspection -->
+                  <shadedPattern>
+                    org.apache.beam.sdk.repackaged.org.apache.commons
+                  </shadedPattern>
+                </relocation>
+              </relocations>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
index 8c2b988e6b03..685efef11845 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
@@ -352,7 +352,15 @@ public ApiSurface includingPackage(String packageName, ClassLoader classLoader)
 
     Set<Class<?>> newRootClasses = Sets.newHashSet();
     for (ClassPath.ClassInfo classInfo : classPath.getTopLevelClassesRecursive(packageName)) {
-      Class clazz = classInfo.load();
+      Class clazz = null;
+      try {
+        clazz = classInfo.load();
+      } catch (NoClassDefFoundError e) {
+        // TODO(BEAM-2231): Ignore any NoClassDefFoundError errors as a workaround.
+        LOG.warn("Failed to load class: {}", classInfo.toString(), e);
+        continue;
+      }
+
       if (exposed(clazz.getModifiers())) {
         newRootClasses.add(clazz);
       }

From 098e6f2b0195fb650d326edb8159ec3688b585f9 Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Tue, 9 May 2017 14:28:24 -0700
Subject: [PATCH 160/387] Fix checkstyle error

---
 .../core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
index 685efef11845..735190b41312 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
@@ -356,7 +356,7 @@ public ApiSurface includingPackage(String packageName, ClassLoader classLoader)
       try {
         clazz = classInfo.load();
       } catch (NoClassDefFoundError e) {
-        // TODO(BEAM-2231): Ignore any NoClassDefFoundError errors as a workaround.
+        // TODO: Ignore any NoClassDefFoundError errors as a workaround. (BEAM-2231)
         LOG.warn("Failed to load class: {}", classInfo.toString(), e);
         continue;
       }

From 9443ee878d77d979dc070b74b0690b569f1635d7 Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Tue, 9 May 2017 17:26:12 -0700
Subject: [PATCH 161/387] Include 'sun.reflect' in GcpCoreApiSurfaceTest

---
 .../apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
index a8772c3b06b3..625c24883331 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
@@ -50,7 +50,8 @@ public void testApiSurface() throws Exception {
             "org.codehaus.jackson",
             // via Avro
             "org.joda.time",
-            "org.junit");
+            "org.junit",
+            "sun.reflect");
 
     assertThat(
         ApiSurface.getSdkApiSurface(getClass().getClassLoader()), containsOnlyPackages(allowed));

From c028349c5ba9d47b6d1d3fb1d765d851c54d2cc1 Mon Sep 17 00:00:00 2001
From: Mark Liu <markliu@google.com>
Date: Tue, 9 May 2017 16:41:46 -0700
Subject: [PATCH 162/387] [BEAM-2236] Move test utilities out of python core

---
 .../apache_beam/coders/standard_coders_test.py      |   2 +-
 .../examples/complete/autocomplete_test.py          |   2 +-
 .../examples/complete/estimate_pi_test.py           |   2 +-
 .../complete/game/hourly_team_score_test.py         |   2 +-
 .../examples/complete/game/user_score_test.py       |   2 +-
 .../apache_beam/examples/complete/tfidf_test.py     |   2 +-
 .../complete/top_wikipedia_sessions_test.py         |   2 +-
 .../examples/cookbook/bigquery_side_input_test.py   |   2 +-
 .../examples/cookbook/bigquery_tornadoes_it_test.py |   4 ++--
 .../examples/cookbook/bigquery_tornadoes_test.py    |   2 +-
 .../apache_beam/examples/cookbook/coders_test.py    |   2 +-
 .../apache_beam/examples/cookbook/combiners_test.py |   2 +-
 .../examples/cookbook/custom_ptransform_test.py     |   2 +-
 .../apache_beam/examples/cookbook/filters_test.py   |   2 +-
 .../apache_beam/examples/snippets/snippets.py       |   2 +-
 .../apache_beam/examples/snippets/snippets_test.py  |   2 +-
 .../apache_beam/examples/wordcount_it_test.py       |   6 +++---
 sdks/python/apache_beam/io/avroio_test.py           |   2 +-
 sdks/python/apache_beam/io/concat_source_test.py    |   2 +-
 sdks/python/apache_beam/io/filebasedsource_test.py  |   6 +++---
 sdks/python/apache_beam/io/fileio_test.py           |   2 +-
 .../apache_beam/io/gcp/datastore/v1/helper_test.py  |   2 +-
 .../apache_beam/io/gcp/tests/bigquery_matcher.py    |   2 +-
 .../io/gcp/tests/bigquery_matcher_test.py           |   2 +-
 sdks/python/apache_beam/io/sources_test.py          |   2 +-
 sdks/python/apache_beam/io/textio_test.py           |   2 +-
 sdks/python/apache_beam/io/tfrecordio_test.py       |   2 +-
 sdks/python/apache_beam/pipeline_test.py            |   2 +-
 sdks/python/apache_beam/pvalue_test.py              |   2 +-
 .../runners/dataflow/dataflow_runner_test.py        |   4 ++--
 .../apache_beam/{tests => testing}/__init__.py      |   0
 .../{tests => testing}/data/privatekey.p12          | Bin
 .../{tests => testing}/data/standard_coders.yaml    |   0
 .../{tests => testing}/pipeline_verifiers.py        |   2 +-
 .../{tests => testing}/pipeline_verifiers_test.py   |   6 +++---
 .../apache_beam/{ => testing}/test_pipeline.py      |   0
 .../apache_beam/{ => testing}/test_pipeline_test.py |   2 +-
 .../apache_beam/{utils => testing}/test_stream.py   |   0
 .../{utils => testing}/test_stream_test.py          |   8 ++++----
 .../apache_beam/{tests => testing}/test_utils.py    |   0
 .../python/apache_beam/transforms/combiners_test.py |   2 +-
 sdks/python/apache_beam/transforms/create_test.py   |   2 +-
 .../apache_beam/transforms/ptransform_test.py       |   2 +-
 .../apache_beam/transforms/sideinputs_test.py       |   2 +-
 sdks/python/apache_beam/transforms/trigger_test.py  |   2 +-
 sdks/python/apache_beam/transforms/util_test.py     |   2 +-
 sdks/python/apache_beam/transforms/window_test.py   |   2 +-
 .../apache_beam/transforms/write_ptransform_test.py |   2 +-
 .../apache_beam/typehints/typed_pipeline_test.py    |   4 ++--
 sdks/python/setup.py                                |   2 +-
 50 files changed, 56 insertions(+), 56 deletions(-)
 rename sdks/python/apache_beam/{tests => testing}/__init__.py (100%)
 rename sdks/python/apache_beam/{tests => testing}/data/privatekey.p12 (100%)
 rename sdks/python/apache_beam/{tests => testing}/data/standard_coders.yaml (100%)
 rename sdks/python/apache_beam/{tests => testing}/pipeline_verifiers.py (98%)
 rename sdks/python/apache_beam/{tests => testing}/pipeline_verifiers_test.py (98%)
 rename sdks/python/apache_beam/{ => testing}/test_pipeline.py (100%)
 rename sdks/python/apache_beam/{ => testing}/test_pipeline_test.py (98%)
 rename sdks/python/apache_beam/{utils => testing}/test_stream.py (100%)
 rename sdks/python/apache_beam/{utils => testing}/test_stream_test.py (92%)
 rename sdks/python/apache_beam/{tests => testing}/test_utils.py (100%)

diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py
index 4a48ed91b108..885e88f6eea5 100644
--- a/sdks/python/apache_beam/coders/standard_coders_test.py
+++ b/sdks/python/apache_beam/coders/standard_coders_test.py
@@ -34,7 +34,7 @@
 from apache_beam.transforms import window
 
 STANDARD_CODERS_YAML = os.path.join(
-    os.path.dirname(__file__), '..', 'tests', 'data', 'standard_coders.yaml')
+    os.path.dirname(__file__), '..', 'testing', 'data', 'standard_coders.yaml')
 
 
 def _load_test_cases(test_yaml):
diff --git a/sdks/python/apache_beam/examples/complete/autocomplete_test.py b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
index d59d0f534f88..438633a25690 100644
--- a/sdks/python/apache_beam/examples/complete/autocomplete_test.py
+++ b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
@@ -21,7 +21,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.complete import autocomplete
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
 
diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
index dc5b90196edc..12d83794fefe 100644
--- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
+++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
@@ -21,7 +21,7 @@
 import unittest
 
 from apache_beam.examples.complete import estimate_pi
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import BeamAssertException
 
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py
index 0eaa8c66d5b0..bd0abcaa1a2b 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py
@@ -21,8 +21,8 @@
 import unittest
 
 import apache_beam as beam
-from apache_beam.test_pipeline import TestPipeline
 from apache_beam.examples.complete.game import hourly_team_score
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class HourlyTeamScoreTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/complete/game/user_score_test.py b/sdks/python/apache_beam/examples/complete/game/user_score_test.py
index 750729dcc99b..2db53bdfb837 100644
--- a/sdks/python/apache_beam/examples/complete/game/user_score_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/user_score_test.py
@@ -21,8 +21,8 @@
 import unittest
 
 import apache_beam as beam
-from apache_beam.test_pipeline import TestPipeline
 from apache_beam.examples.complete.game import user_score
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class UserScoreTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/complete/tfidf_test.py b/sdks/python/apache_beam/examples/complete/tfidf_test.py
index 05e53a4ba3f9..0e3025459445 100644
--- a/sdks/python/apache_beam/examples/complete/tfidf_test.py
+++ b/sdks/python/apache_beam/examples/complete/tfidf_test.py
@@ -25,7 +25,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.complete import tfidf
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 EXPECTED_RESULTS = set([
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
index 9b9d9b1d0bc4..4850c04cee98 100644
--- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
+++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
@@ -23,7 +23,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.complete import top_wikipedia_sessions
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class ComputeTopSessionsTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
index 586997648145..1ca25c9e14e9 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
@@ -22,7 +22,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.cookbook import bigquery_side_input
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class BigQuerySideInputTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
index 3e302d1ede60..5d2ee7c2cef8 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
@@ -26,8 +26,8 @@
 
 from apache_beam.examples.cookbook import bigquery_tornadoes
 from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher
-from apache_beam.test_pipeline import TestPipeline
-from apache_beam.tests.pipeline_verifiers import PipelineStateMatcher
+from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class BigqueryTornadoesIT(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
index 0c66d7e198f9..ca7ca9e2e74d 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
@@ -22,7 +22,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.cookbook import bigquery_tornadoes
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class BigQueryTornadoesTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/cookbook/coders_test.py b/sdks/python/apache_beam/examples/cookbook/coders_test.py
index 4a92abbdca24..35cf252575d4 100644
--- a/sdks/python/apache_beam/examples/cookbook/coders_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/coders_test.py
@@ -22,7 +22,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.cookbook import coders
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
 
diff --git a/sdks/python/apache_beam/examples/cookbook/combiners_test.py b/sdks/python/apache_beam/examples/cookbook/combiners_test.py
index a8ed5550ccc8..45c779fd5159 100644
--- a/sdks/python/apache_beam/examples/cookbook/combiners_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/combiners_test.py
@@ -27,7 +27,7 @@
 import unittest
 
 import apache_beam as beam
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class CombinersTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
index cd1c04ac6cdb..2d35d8d29d2a 100644
--- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
@@ -22,7 +22,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.cookbook import custom_ptransform
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
 
diff --git a/sdks/python/apache_beam/examples/cookbook/filters_test.py b/sdks/python/apache_beam/examples/cookbook/filters_test.py
index 28bb1e14aecd..44a352f19e64 100644
--- a/sdks/python/apache_beam/examples/cookbook/filters_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/filters_test.py
@@ -22,7 +22,7 @@
 
 import apache_beam as beam
 from apache_beam.examples.cookbook import filters
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class FiltersTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py
index 26af71da883e..1bdb9a366bf0 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets.py
@@ -31,8 +31,8 @@
 """
 
 import apache_beam as beam
-from apache_beam.test_pipeline import TestPipeline
 from apache_beam.metrics import Metrics
+from apache_beam.testing.test_pipeline import TestPipeline
 
 # Quiet some pylint warnings that happen because of the somewhat special
 # format for the code snippets.
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index da0a9625cfda..85d8bde186fb 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -36,7 +36,7 @@
 from apache_beam.utils.windowed_value import WindowedValue
 
 # pylint: disable=expression-not-assigned
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 # Protect against environments where apitools library is not available.
 # pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py
index 54e54e8b00da..4bee1279c8f0 100644
--- a/sdks/python/apache_beam/examples/wordcount_it_test.py
+++ b/sdks/python/apache_beam/examples/wordcount_it_test.py
@@ -25,9 +25,9 @@
 from nose.plugins.attrib import attr
 
 from apache_beam.examples import wordcount
-from apache_beam.test_pipeline import TestPipeline
-from apache_beam.tests.pipeline_verifiers import PipelineStateMatcher
-from apache_beam.tests.pipeline_verifiers import FileChecksumMatcher
+from apache_beam.testing.pipeline_verifiers import FileChecksumMatcher
+from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class WordCountIT(unittest.TestCase):
diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py
index 5f2db6268113..4a2183928b98 100644
--- a/sdks/python/apache_beam/io/avroio_test.py
+++ b/sdks/python/apache_beam/io/avroio_test.py
@@ -26,7 +26,7 @@
 from apache_beam.io import avroio
 from apache_beam.io import filebasedsource
 from apache_beam.io import source_test_utils
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
 from apache_beam.transforms.util import assert_that
diff --git a/sdks/python/apache_beam/io/concat_source_test.py b/sdks/python/apache_beam/io/concat_source_test.py
index 807c3fd8e4ce..a02f9ad64663 100644
--- a/sdks/python/apache_beam/io/concat_source_test.py
+++ b/sdks/python/apache_beam/io/concat_source_test.py
@@ -26,7 +26,7 @@
 from apache_beam.io import range_trackers
 from apache_beam.io import source_test_utils
 from apache_beam.io.concat_source import ConcatSource
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
 
diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py
index 4ff23fc3d6a4..e17a004a3430 100644
--- a/sdks/python/apache_beam/io/filebasedsource_test.py
+++ b/sdks/python/apache_beam/io/filebasedsource_test.py
@@ -38,13 +38,13 @@
 from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource
 
 from apache_beam.io.filebasedsource import FileBasedSource
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.options.value_provider import StaticValueProvider
+from apache_beam.options.value_provider import RuntimeValueProvider
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
-from apache_beam.options.value_provider import StaticValueProvider
-from apache_beam.options.value_provider import RuntimeValueProvider
 
 
 class LineSource(FileBasedSource):
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index e0e9774dd255..4c25505d5212 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -31,7 +31,7 @@
 import apache_beam as beam
 from apache_beam import coders
 from apache_beam.io import fileio
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
 
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py
index 5d4bb6f6495c..a804c09c7495 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py
@@ -23,7 +23,7 @@
 
 from apache_beam.io.gcp.datastore.v1 import fake_datastore
 from apache_beam.io.gcp.datastore.v1 import helper
-from apache_beam.tests.test_utils import patch_retry
+from apache_beam.testing.test_utils import patch_retry
 
 
 # Protect against environments where apitools library is not available.
diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
index 66d99b3ab7aa..f42b70fc19f4 100644
--- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
+++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
@@ -21,7 +21,7 @@
 
 from hamcrest.core.base_matcher import BaseMatcher
 
-from apache_beam.tests.test_utils import compute_hash
+from apache_beam.testing.test_utils import compute_hash
 from apache_beam.utils import retry
 
 # Protect against environments where bigquery library is not available.
diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py
index d8aa1489dc15..f12293e491c8 100644
--- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py
+++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py
@@ -24,7 +24,7 @@
 from mock import Mock, patch
 
 from apache_beam.io.gcp.tests import bigquery_matcher as bq_verifier
-from apache_beam.tests.test_utils import patch_retry
+from apache_beam.testing.test_utils import patch_retry
 
 # Protect against environments where bigquery library is not available.
 # pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/io/sources_test.py b/sdks/python/apache_beam/io/sources_test.py
index 3f92756979b3..c0b8ad620dc8 100644
--- a/sdks/python/apache_beam/io/sources_test.py
+++ b/sdks/python/apache_beam/io/sources_test.py
@@ -27,7 +27,7 @@
 from apache_beam import coders
 from apache_beam.io import iobase
 from apache_beam.io import range_trackers
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
 
diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py
index 90dc6652fa47..d00afef6c033 100644
--- a/sdks/python/apache_beam/io/textio_test.py
+++ b/sdks/python/apache_beam/io/textio_test.py
@@ -43,7 +43,7 @@
 from apache_beam.io.filebasedsource_test import write_pattern
 from apache_beam.io.filesystem import CompressionTypes
 
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
diff --git a/sdks/python/apache_beam/io/tfrecordio_test.py b/sdks/python/apache_beam/io/tfrecordio_test.py
index 29a9fb8342a0..b7e370d0187a 100644
--- a/sdks/python/apache_beam/io/tfrecordio_test.py
+++ b/sdks/python/apache_beam/io/tfrecordio_test.py
@@ -35,7 +35,7 @@
 from apache_beam.io.tfrecordio import _TFRecordUtil
 from apache_beam.io.tfrecordio import ReadFromTFRecord
 from apache_beam.io.tfrecordio import WriteToTFRecord
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 import crcmod
 
 
diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py
index ebcc43b5b2fb..c6b1e48fb5b5 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -32,7 +32,7 @@
 from apache_beam.pipeline import PipelineVisitor
 from apache_beam.pvalue import AsSingleton
 from apache_beam.runners.dataflow.native_io.iobase import NativeSource
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms import CombineGlobally
 from apache_beam.transforms import Create
 from apache_beam.transforms import FlatMap
diff --git a/sdks/python/apache_beam/pvalue_test.py b/sdks/python/apache_beam/pvalue_test.py
index 529ddf773c2c..4acbc52c2237 100644
--- a/sdks/python/apache_beam/pvalue_test.py
+++ b/sdks/python/apache_beam/pvalue_test.py
@@ -20,7 +20,7 @@
 import unittest
 
 from apache_beam.pvalue import PValue
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class PValueTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
index a9f61a7ce7d7..b61a683f9689 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
@@ -26,6 +26,7 @@
 import apache_beam as beam
 import apache_beam.transforms as ptransform
 
+from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.pipeline import Pipeline, AppliedPTransform
 from apache_beam.pvalue import PCollection
 from apache_beam.runners import create_runner
@@ -34,10 +35,9 @@
 from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult
 from apache_beam.runners.dataflow.dataflow_runner import DataflowRuntimeException
 from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayDataItem
 from apache_beam.typehints import typehints
-from apache_beam.options.pipeline_options import PipelineOptions
 
 # Protect against environments where apitools library is not available.
 # pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/tests/__init__.py b/sdks/python/apache_beam/testing/__init__.py
similarity index 100%
rename from sdks/python/apache_beam/tests/__init__.py
rename to sdks/python/apache_beam/testing/__init__.py
diff --git a/sdks/python/apache_beam/tests/data/privatekey.p12 b/sdks/python/apache_beam/testing/data/privatekey.p12
similarity index 100%
rename from sdks/python/apache_beam/tests/data/privatekey.p12
rename to sdks/python/apache_beam/testing/data/privatekey.p12
diff --git a/sdks/python/apache_beam/tests/data/standard_coders.yaml b/sdks/python/apache_beam/testing/data/standard_coders.yaml
similarity index 100%
rename from sdks/python/apache_beam/tests/data/standard_coders.yaml
rename to sdks/python/apache_beam/testing/data/standard_coders.yaml
diff --git a/sdks/python/apache_beam/tests/pipeline_verifiers.py b/sdks/python/apache_beam/testing/pipeline_verifiers.py
similarity index 98%
rename from sdks/python/apache_beam/tests/pipeline_verifiers.py
rename to sdks/python/apache_beam/testing/pipeline_verifiers.py
index df05054c22dd..5a6082aaee50 100644
--- a/sdks/python/apache_beam/tests/pipeline_verifiers.py
+++ b/sdks/python/apache_beam/testing/pipeline_verifiers.py
@@ -29,7 +29,7 @@
 
 from apache_beam.io.filesystems import FileSystems
 from apache_beam.runners.runner import PipelineState
-from apache_beam.tests import test_utils as utils
+from apache_beam.testing import test_utils as utils
 from apache_beam.utils import retry
 
 try:
diff --git a/sdks/python/apache_beam/tests/pipeline_verifiers_test.py b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py
similarity index 98%
rename from sdks/python/apache_beam/tests/pipeline_verifiers_test.py
rename to sdks/python/apache_beam/testing/pipeline_verifiers_test.py
index 909917dc53cf..15e0a042008b 100644
--- a/sdks/python/apache_beam/tests/pipeline_verifiers_test.py
+++ b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py
@@ -25,10 +25,10 @@
 from mock import Mock, patch
 
 from apache_beam.io.localfilesystem import LocalFileSystem
-from apache_beam.runners.runner import PipelineState
 from apache_beam.runners.runner import PipelineResult
-from apache_beam.tests import pipeline_verifiers as verifiers
-from apache_beam.tests.test_utils import patch_retry
+from apache_beam.runners.runner import PipelineState
+from apache_beam.testing.test_utils import patch_retry
+from apache_beam.testing import pipeline_verifiers as verifiers
 
 try:
   # pylint: disable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py
similarity index 100%
rename from sdks/python/apache_beam/test_pipeline.py
rename to sdks/python/apache_beam/testing/test_pipeline.py
diff --git a/sdks/python/apache_beam/test_pipeline_test.py b/sdks/python/apache_beam/testing/test_pipeline_test.py
similarity index 98%
rename from sdks/python/apache_beam/test_pipeline_test.py
rename to sdks/python/apache_beam/testing/test_pipeline_test.py
index 325cab789518..747d64c7409f 100644
--- a/sdks/python/apache_beam/test_pipeline_test.py
+++ b/sdks/python/apache_beam/testing/test_pipeline_test.py
@@ -24,7 +24,7 @@
 from hamcrest.core.assert_that import assert_that as hc_assert_that
 
 from apache_beam.internal import pickler
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.options.pipeline_options import PipelineOptions
 
 
diff --git a/sdks/python/apache_beam/utils/test_stream.py b/sdks/python/apache_beam/testing/test_stream.py
similarity index 100%
rename from sdks/python/apache_beam/utils/test_stream.py
rename to sdks/python/apache_beam/testing/test_stream.py
diff --git a/sdks/python/apache_beam/utils/test_stream_test.py b/sdks/python/apache_beam/testing/test_stream_test.py
similarity index 92%
rename from sdks/python/apache_beam/utils/test_stream_test.py
rename to sdks/python/apache_beam/testing/test_stream_test.py
index b5b5c697e554..e32dda2aeb26 100644
--- a/sdks/python/apache_beam/utils/test_stream_test.py
+++ b/sdks/python/apache_beam/testing/test_stream_test.py
@@ -19,12 +19,12 @@
 
 import unittest
 
+from apache_beam.testing.test_stream import ElementEvent
+from apache_beam.testing.test_stream import ProcessingTimeEvent
+from apache_beam.testing.test_stream import TestStream
+from apache_beam.testing.test_stream import WatermarkEvent
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.utils import timestamp
-from apache_beam.utils.test_stream import ElementEvent
-from apache_beam.utils.test_stream import ProcessingTimeEvent
-from apache_beam.utils.test_stream import TestStream
-from apache_beam.utils.test_stream import WatermarkEvent
 from apache_beam.utils.windowed_value import WindowedValue
 
 
diff --git a/sdks/python/apache_beam/tests/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py
similarity index 100%
rename from sdks/python/apache_beam/tests/test_utils.py
rename to sdks/python/apache_beam/testing/test_utils.py
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
index af768891c84f..1822c1911927 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -22,7 +22,7 @@
 import hamcrest as hc
 
 import apache_beam as beam
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 import apache_beam.transforms.combiners as combine
 from apache_beam.transforms.core import CombineGlobally
 from apache_beam.transforms.core import Create
diff --git a/sdks/python/apache_beam/transforms/create_test.py b/sdks/python/apache_beam/transforms/create_test.py
index 2352acd3c53f..9ede4c7d2d41 100644
--- a/sdks/python/apache_beam/transforms/create_test.py
+++ b/sdks/python/apache_beam/transforms/create_test.py
@@ -22,7 +22,7 @@
 
 from apache_beam import Create, assert_that, equal_to
 from apache_beam.coders import FastPrimitivesCoder
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 
 
 class CreateTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index 5948460579de..137992dd428b 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -32,7 +32,7 @@
 from apache_beam.io.iobase import Read
 from apache_beam.options.pipeline_options import TypeOptions
 import apache_beam.pvalue as pvalue
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms import window
 import apache_beam.transforms.combiners as combine
 from apache_beam.transforms.display import DisplayData, DisplayDataItem
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py
index bf9aeff084ee..0bc910799571 100644
--- a/sdks/python/apache_beam/transforms/sideinputs_test.py
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -23,7 +23,7 @@
 from nose.plugins.attrib import attr
 
 import apache_beam as beam
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms import window
 from apache_beam.transforms.util import assert_that, equal_to
 
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index c08a791bab4e..da7f43bdf82a 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -26,7 +26,7 @@
 
 import apache_beam as beam
 from apache_beam.runners import pipeline_context
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms import trigger
 from apache_beam.transforms.core import Windowing
 from apache_beam.transforms.trigger import AccumulationMode
diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py
index 965682786321..7fdef702871f 100644
--- a/sdks/python/apache_beam/transforms/util_test.py
+++ b/sdks/python/apache_beam/transforms/util_test.py
@@ -20,7 +20,7 @@
 import unittest
 
 from apache_beam import Create
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that, equal_to, is_empty
 
 
diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py
index 2d2b03d90ed8..a7797dd435d9 100644
--- a/sdks/python/apache_beam/transforms/window_test.py
+++ b/sdks/python/apache_beam/transforms/window_test.py
@@ -20,7 +20,7 @@
 import unittest
 
 from apache_beam.runners import pipeline_context
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms import CombinePerKey
 from apache_beam.transforms import combiners
 from apache_beam.transforms import core
diff --git a/sdks/python/apache_beam/transforms/write_ptransform_test.py b/sdks/python/apache_beam/transforms/write_ptransform_test.py
index 3d7fbd94f336..27e7caa8e5a7 100644
--- a/sdks/python/apache_beam/transforms/write_ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/write_ptransform_test.py
@@ -22,7 +22,7 @@
 import apache_beam as beam
 
 from apache_beam.io import iobase
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.ptransform import PTransform
 from apache_beam.transforms.util import assert_that, is_empty
 
diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
index 3812fb1270e2..3494cfe1e6a8 100644
--- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
+++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
@@ -23,10 +23,10 @@
 import apache_beam as beam
 from apache_beam import pvalue
 from apache_beam import typehints
-from apache_beam.test_pipeline import TestPipeline
+from apache_beam.options.pipeline_options import OptionsContext
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.util import assert_that, equal_to
 from apache_beam.typehints import WithTypeHints
-from apache_beam.options.pipeline_options import OptionsContext
 
 # These test often construct a pipeline as value | PTransform to test side
 # effects (e.g. errors).
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 681abbfce96b..9bf3cf4d37e9 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -122,7 +122,7 @@ def get_version():
     author_email=PACKAGE_EMAIL,
     packages=setuptools.find_packages(),
     package_data={'apache_beam': [
-        '*/*.pyx', '*/*/*.pyx', '*/*.pxd', '*/*/*.pxd', 'tests/data/*']},
+        '*/*.pyx', '*/*/*.pyx', '*/*.pxd', '*/*/*.pxd', 'testing/data/*']},
     ext_modules=cythonize([
         'apache_beam/**/*.pyx',
         'apache_beam/coders/coder_impl.py',

From 3499c70a2fde0124dfc48ae20281c2261ef9018a Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 18:32:14 -0700
Subject: [PATCH 163/387] Mark PValue and PValueBase Internal

These should not be referred to as their abstract types by users, and
PipelineRunners won't interact with PValues that aren't PCollections or
PCollectionViews.
---
 .../java/org/apache/beam/sdk/values/PValue.java   | 12 +++++++-----
 .../org/apache/beam/sdk/values/PValueBase.java    | 15 +++++----------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java
index d9f692014d86..108902879236 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValue.java
@@ -18,13 +18,15 @@
 package org.apache.beam.sdk.values;
 
 import java.util.Map;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
- * The interface for values that can be input to and output from {@link PTransform PTransforms}.
+ * <b><i>For internal use. No backwards compatibility guarantees.</i></b>
  *
- * <p>It is recommended to extend {@link PValueBase}
+ * <p>A primitive value within Beam.
  */
+@Internal
 public interface PValue extends POutput, PInput {
 
   /**
@@ -45,13 +47,13 @@ public interface PValue extends POutput, PInput {
    * After building, finalizes this {@code PValue} to make it ready for being used as an input to a
    * {@link org.apache.beam.sdk.transforms.PTransform}.
    *
-   * <p>Automatically invoked whenever {@code apply()} is invoked on this {@code PValue}, after
-   * {@link PValue#finishSpecifying(PInput, PTransform)} has been called on each component {@link
-   * PValue}, so users do not normally call this explicitly.
+   * <p>Automatically invoked whenever {@code apply()} is invoked on this {@code PValue}. Users
+   * should not normally call this explicitly.
    *
    * @param upstreamInput the {@link PInput} the {@link PTransform} was applied to to produce this
    *     output
    * @param upstreamTransform the {@link PTransform} that produced this {@link PValue}
    */
+  @Internal
   void finishSpecifying(PInput upstreamInput, PTransform<?, ?> upstreamTransform);
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
index 4de0589d1222..6f638d7b9a3a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
@@ -22,22 +22,17 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.NameUtils;
 
 /**
- * A {@link PValueBase} is an abstract base class that provides
- * sensible default implementations for methods of {@link PValue}.
- * In particular, this includes functionality for getting/setting:
+ * <b><i>For internal use. No backwards compatibility guarantees.</i></b>
  *
- * <ul>
- *   <li> The {@link Pipeline} that the {@link PValue} is part of.</li>
- *   <li> Whether the {@link PValue} has bee finalized (as an input
- *     or an output), after which its properties can no longer be changed.</li>
- * </ul>
- *
- * <p>For internal use.
+ * <p>An abstract base class that provides default implementations for some methods of
+ * {@link PValue}.
  */
+@Internal
 public abstract class PValueBase implements PValue {
 
   private final Pipeline pipeline;

From 12c9ba1a95204daaf089de93b615ae9cd8a6c4fa Mon Sep 17 00:00:00 2001
From: Maria Garcia Herrero <mariagh@google.com>
Date: Tue, 9 May 2017 17:43:33 -0700
Subject: [PATCH 164/387] Mark PipelineVisitor and AppliedPTransform as
 internal.

---
 sdks/python/apache_beam/pipeline.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index ed896c4fd0d1..83c72877c701 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -361,9 +361,10 @@ def from_runner_api(proto, runner, options):
 
 
 class PipelineVisitor(object):
-  """Visitor pattern class used to traverse a DAG of transforms.
+  """For internal use only; no backwards-compatibility guarantees.
 
-  This is an internal class used for bookkeeping by a Pipeline.
+  Visitor pattern class used to traverse a DAG of transforms
+  (used internally by Pipeline for bookeeping purposes).
   """
 
   def visit_value(self, value, producer_node):
@@ -390,9 +391,10 @@ def leave_composite_transform(self, transform_node):
 
 
 class AppliedPTransform(object):
-  """A transform node representing an instance of applying a PTransform.
+  """For internal use only; no backwards-compatibility guarantees.
 
-  This is an internal class used for bookkeeping by a Pipeline.
+  A transform node representing an instance of applying a PTransform
+  (used internally by Pipeline for bookeeping purposes).
   """
 
   def __init__(self, parent, transform, full_label, inputs):

From cc3d07aadaa88841e039f8bc15b3b979fca77d22 Mon Sep 17 00:00:00 2001
From: "chamikara@google.com" <chamikara@google.com>
Date: Tue, 9 May 2017 19:56:14 -0700
Subject: [PATCH 165/387] Renames some python classes and functions that were
 unnecessarily public.

Adds a note to documentation of classes that are public but should be only used internally by the SDK (non-user facing classes).

Marks some of the modules as experimental.
---
 sdks/python/apache_beam/coders/coder_impl.py  | 57 +++++++++++----
 sdks/python/apache_beam/coders/observable.py  |  4 +-
 sdks/python/apache_beam/coders/slow_stream.py | 16 +++--
 sdks/python/apache_beam/coders/typecoders.py  |  4 +-
 sdks/python/apache_beam/internal/gcp/auth.py  | 18 +++--
 .../apache_beam/internal/gcp/json_value.py    | 12 +++-
 sdks/python/apache_beam/internal/pickler.py   | 26 ++++---
 sdks/python/apache_beam/internal/util.py      | 16 +++--
 sdks/python/apache_beam/io/concat_source.py   |  8 ++-
 sdks/python/apache_beam/metrics/cells.py      | 32 ++++++---
 sdks/python/apache_beam/metrics/execution.py  |  7 +-
 .../runners/api/beam_fn_api_pb2.py            |  2 +
 .../runners/api/beam_fn_api_pb2_grpc.py       |  2 +
 .../runners/api/beam_runner_api_pb2.py        |  2 +
 sdks/python/apache_beam/runners/common.pxd    |  4 +-
 sdks/python/apache_beam/runners/common.py     | 28 +++++---
 .../runners/dataflow/internal/apiclient.py    |  4 +-
 .../runners/dataflow/internal/dependency.py   | 16 +++--
 .../runners/dataflow/internal/names.py        |  8 ++-
 .../runners/dataflow/native_io/iobase.py      |  2 +
 .../runners/direct/bundle_factory.py          | 16 +++--
 .../apache_beam/runners/direct/clock.py       |  5 +-
 .../consumer_tracking_pipeline_visitor.py     |  4 +-
 .../apache_beam/runners/direct/executor.py    | 63 ++++++++--------
 .../runners/direct/transform_evaluator.py     |  8 ++-
 .../runners/direct/transform_result.py        |  4 +-
 .../runners/direct/watermark_manager.py       | 10 +--
 .../apache_beam/runners/pipeline_context.py   |  4 +-
 .../runners/portability/fn_api_runner.py      |  2 +
 .../portability/maptask_executor_runner.py    |  2 +
 sdks/python/apache_beam/runners/runner.py     |  8 ++-
 .../apache_beam/runners/worker/data_plane.py  |  2 +
 .../apache_beam/runners/worker/log_handler.py |  2 +
 .../apache_beam/runners/worker/logger.py      |  2 +
 .../apache_beam/runners/worker/opcounters.py  |  2 +
 .../runners/worker/operation_specs.py         |  2 +
 .../apache_beam/runners/worker/sdk_worker.py  |  4 ++
 .../runners/worker/sdk_worker_main.py         |  2 +
 .../apache_beam/runners/worker/sideinputs.py  |  2 +
 .../runners/worker/statesampler_fake.py       |  2 +
 sdks/python/apache_beam/transforms/core.py    | 12 +++-
 .../apache_beam/transforms/ptransform.py      | 14 ++--
 sdks/python/apache_beam/transforms/trigger.py | 72 +++++++++----------
 43 files changed, 346 insertions(+), 166 deletions(-)

diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py
index d56606d3962d..a0496a2bbfb5 100644
--- a/sdks/python/apache_beam/coders/coder_impl.py
+++ b/sdks/python/apache_beam/coders/coder_impl.py
@@ -50,6 +50,7 @@
 
 
 class CoderImpl(object):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def encode_to_stream(self, value, stream, nested):
     """Reads object from potentially-nested encoding in stream."""
@@ -97,7 +98,9 @@ def get_estimated_size_and_observables(self, value, nested=False):
 
 
 class SimpleCoderImpl(CoderImpl):
-  """Subclass of CoderImpl implementing stream methods using encode/decode."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Subclass of CoderImpl implementing stream methods using encode/decode."""
 
   def encode_to_stream(self, value, stream, nested):
     """Reads object from potentially-nested encoding in stream."""
@@ -109,7 +112,9 @@ def decode_from_stream(self, stream, nested):
 
 
 class StreamCoderImpl(CoderImpl):
-  """Subclass of CoderImpl implementing encode/decode using stream methods."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Subclass of CoderImpl implementing encode/decode using stream methods."""
 
   def encode(self, value):
     out = create_OutputStream()
@@ -127,7 +132,9 @@ def estimate_size(self, value, nested=False):
 
 
 class CallbackCoderImpl(CoderImpl):
-  """A CoderImpl that calls back to the _impl methods on the Coder itself.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A CoderImpl that calls back to the _impl methods on the Coder itself.
 
   This is the default implementation used if Coder._get_impl()
   is not overwritten.
@@ -166,6 +173,7 @@ def get_estimated_size_and_observables(self, value, nested=False):
 
 
 class DeterministicFastPrimitivesCoderImpl(CoderImpl):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def __init__(self, coder, step_label):
     self._underlying_coder = coder
@@ -208,6 +216,7 @@ def get_estimated_size_and_observables(self, value, nested=False):
 
 
 class ProtoCoderImpl(SimpleCoderImpl):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def __init__(self, proto_message_type):
     self.proto_message_type = proto_message_type
@@ -235,6 +244,7 @@ def decode(self, encoded):
 
 
 class FastPrimitivesCoderImpl(StreamCoderImpl):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def __init__(self, fallback_coder_impl):
     self.fallback_coder_impl = fallback_coder_impl
@@ -319,7 +329,9 @@ def decode_from_stream(self, stream, nested):
 
 
 class BytesCoderImpl(CoderImpl):
-  """A coder for bytes/str objects."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A coder for bytes/str objects."""
 
   def encode_to_stream(self, value, out, nested):
     out.write(value, nested)
@@ -336,6 +348,7 @@ def decode(self, encoded):
 
 
 class FloatCoderImpl(StreamCoderImpl):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def encode_to_stream(self, value, out, nested):
     out.write_bigendian_double(value)
@@ -349,6 +362,8 @@ def estimate_size(self, unused_value, nested=False):
 
 
 class IntervalWindowCoderImpl(StreamCoderImpl):
+  """For internal use only; no backwards-compatibility guarantees."""
+
   # TODO: Fn Harness only supports millis. Is this important enough to fix?
   def _to_normal_time(self, value):
     """Convert "lexicographically ordered unsigned" to signed."""
@@ -379,6 +394,8 @@ def estimate_size(self, value, nested=False):
 
 
 class TimestampCoderImpl(StreamCoderImpl):
+  """For internal use only; no backwards-compatibility guarantees."""
+
   def encode_to_stream(self, value, out, nested):
     out.write_bigendian_int64(value.micros)
 
@@ -395,7 +412,9 @@ def estimate_size(self, unused_value, nested=False):
 
 
 class VarIntCoderImpl(StreamCoderImpl):
-  """A coder for long/int objects."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A coder for long/int objects."""
 
   def encode_to_stream(self, value, out, nested):
     out.write_var_int64(value)
@@ -422,7 +441,9 @@ def estimate_size(self, value, nested=False):
 
 
 class SingletonCoderImpl(CoderImpl):
-  """A coder that always encodes exactly one value."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A coder that always encodes exactly one value."""
 
   def __init__(self, value):
     self._value = value
@@ -445,7 +466,9 @@ def estimate_size(self, value, nested=False):
 
 
 class AbstractComponentCoderImpl(StreamCoderImpl):
-  """CoderImpl for coders that are comprised of several component coders."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  CoderImpl for coders that are comprised of several component coders."""
 
   def __init__(self, coder_impls):
     for c in coder_impls:
@@ -507,7 +530,9 @@ def _construct_from_components(self, components):
 
 
 class SequenceCoderImpl(StreamCoderImpl):
-  """A coder for sequences.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A coder for sequences.
 
   If the length of the sequence in known we encode the length as a 32 bit
   ``int`` followed by the encoded bytes.
@@ -611,21 +636,27 @@ def get_estimated_size_and_observables(self, value, nested=False):
 
 
 class TupleSequenceCoderImpl(SequenceCoderImpl):
-  """A coder for homogeneous tuple objects."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A coder for homogeneous tuple objects."""
 
   def _construct_from_sequence(self, components):
     return tuple(components)
 
 
 class IterableCoderImpl(SequenceCoderImpl):
-  """A coder for homogeneous iterable objects."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A coder for homogeneous iterable objects."""
 
   def _construct_from_sequence(self, components):
     return components
 
 
 class WindowedValueCoderImpl(StreamCoderImpl):
-  """A coder for windowed values."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A coder for windowed values."""
 
   # Ensure that lexicographic ordering of the bytes corresponds to
   # chronological order of timestamps.
@@ -713,7 +744,9 @@ def get_estimated_size_and_observables(self, value, nested=False):
 
 
 class LengthPrefixCoderImpl(StreamCoderImpl):
-  """Coder which prefixes the length of the encoded object in the stream."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Coder which prefixes the length of the encoded object in the stream."""
 
   def __init__(self, value_coder):
     self._value_coder = value_coder
diff --git a/sdks/python/apache_beam/coders/observable.py b/sdks/python/apache_beam/coders/observable.py
index f344b5d0d38e..5a808d8ad38c 100644
--- a/sdks/python/apache_beam/coders/observable.py
+++ b/sdks/python/apache_beam/coders/observable.py
@@ -20,7 +20,9 @@
 
 
 class ObservableMixin(object):
-  """An observable iterable.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  An observable iterable.
 
   Subclasses need to call self.notify_observers with any object yielded.
   """
diff --git a/sdks/python/apache_beam/coders/slow_stream.py b/sdks/python/apache_beam/coders/slow_stream.py
index a87495c94bfe..85837bcfa002 100644
--- a/sdks/python/apache_beam/coders/slow_stream.py
+++ b/sdks/python/apache_beam/coders/slow_stream.py
@@ -21,7 +21,9 @@
 
 
 class OutputStream(object):
-  """A pure Python implementation of stream.OutputStream."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A pure Python implementation of stream.OutputStream."""
 
   def __init__(self):
     self.data = []
@@ -69,7 +71,9 @@ def size(self):
 
 
 class ByteCountingOutputStream(OutputStream):
-  """A pure Python implementation of stream.ByteCountingOutputStream."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A pure Python implementation of stream.ByteCountingOutputStream."""
 
   def __init__(self):
     # Note that we don't actually use any of the data initialized by our super.
@@ -96,7 +100,9 @@ def __str__(self):
 
 
 class InputStream(object):
-  """A pure Python implementation of stream.InputStream."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A pure Python implementation of stream.InputStream."""
 
   def __init__(self, data):
     self.data = data
@@ -149,7 +155,9 @@ def read_bigendian_double(self):
 
 
 def get_varint_size(v):
-  """Returns the size of the given integer value when encode as a VarInt."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Returns the size of the given integer value when encode as a VarInt."""
   if v < 0:
     v += 1 << 64
     if v <= 0:
diff --git a/sdks/python/apache_beam/coders/typecoders.py b/sdks/python/apache_beam/coders/typecoders.py
index 1bd4312705f2..60832c9b130c 100644
--- a/sdks/python/apache_beam/coders/typecoders.py
+++ b/sdks/python/apache_beam/coders/typecoders.py
@@ -160,7 +160,9 @@ def verify_deterministic(self, key_coder, op_name, silent=True):
 
 
 class FirstOf(object):
-  """A class used to get the first matching coder from a list of coders."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A class used to get the first matching coder from a list of coders."""
 
   def __init__(self, coders):
     self._coders = coders
diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py
index 3c8dd64d9759..9f32092f4166 100644
--- a/sdks/python/apache_beam/internal/gcp/auth.py
+++ b/sdks/python/apache_beam/internal/gcp/auth.py
@@ -38,7 +38,9 @@
 
 
 def set_running_in_gce(worker_executing_project):
-  """Informs the authentication library that we are running in GCE.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Informs the authentication library that we are running in GCE.
 
   When we are running in GCE, we have the option of using the VM metadata
   credentials for authentication to Google services.
@@ -57,8 +59,10 @@ class AuthenticationException(retry.PermanentException):
   pass
 
 
-class GCEMetadataCredentials(OAuth2Credentials):
-  """Credential object initialized using access token from GCE VM metadata."""
+class _GCEMetadataCredentials(OAuth2Credentials):
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Credential object initialized using access token from GCE VM metadata."""
 
   def __init__(self, user_agent=None):
     """Create an instance of GCEMetadataCredentials.
@@ -69,7 +73,7 @@ def __init__(self, user_agent=None):
     Args:
       user_agent: string, The HTTP User-Agent to provide for this application.
     """
-    super(GCEMetadataCredentials, self).__init__(
+    super(_GCEMetadataCredentials, self).__init__(
         None,  # access_token
         None,  # client_id
         None,  # client_secret
@@ -94,7 +98,9 @@ def _refresh(self, http_request):
 
 
 def get_service_credentials():
-  """Get credentials to access Google services."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Get credentials to access Google services."""
   user_agent = 'beam-python-sdk/1.0'
   if is_running_in_gce:
     # We are currently running as a GCE taskrunner worker.
@@ -102,7 +108,7 @@ def get_service_credentials():
     # TODO(ccy): It's not entirely clear if these credentials are thread-safe.
     # If so, we can cache these credentials to save the overhead of creating
     # them again.
-    return GCEMetadataCredentials(user_agent=user_agent)
+    return _GCEMetadataCredentials(user_agent=user_agent)
   else:
     client_scopes = [
         'https://www.googleapis.com/auth/bigquery',
diff --git a/sdks/python/apache_beam/internal/gcp/json_value.py b/sdks/python/apache_beam/internal/gcp/json_value.py
index 523db9c7d804..59f8b60dd9d7 100644
--- a/sdks/python/apache_beam/internal/gcp/json_value.py
+++ b/sdks/python/apache_beam/internal/gcp/json_value.py
@@ -33,7 +33,9 @@
 
 
 def get_typed_value_descriptor(obj):
-  """Converts a basic type into a @type/value dictionary.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Converts a basic type into a @type/value dictionary.
 
   Args:
     obj: A basestring, bool, int, or float to be converted.
@@ -59,7 +61,9 @@ def get_typed_value_descriptor(obj):
 
 
 def to_json_value(obj, with_type=False):
-  """Converts Python objects into extra_types.JsonValue objects.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Converts Python objects into extra_types.JsonValue objects.
 
   Args:
     obj: Python object to be converted. Can be 'None'.
@@ -115,7 +119,9 @@ def to_json_value(obj, with_type=False):
 
 
 def from_json_value(v):
-  """Converts extra_types.JsonValue objects into Python objects.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Converts extra_types.JsonValue objects into Python objects.
 
   Args:
     v: JsonValue object to be converted.
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index ed429fe084be..4305379747b3 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -36,25 +36,25 @@
 import dill
 
 
-def is_nested_class(cls):
+def _is_nested_class(cls):
   """Returns true if argument is a class object that appears to be nested."""
   return (isinstance(cls, type)
           and cls.__module__ != '__builtin__'
           and cls.__name__ not in sys.modules[cls.__module__].__dict__)
 
 
-def find_containing_class(nested_class):
+def _find_containing_class(nested_class):
   """Finds containing class of a nestec class passed as argument."""
 
-  def find_containing_class_inner(outer):
+  def _find_containing_class_inner(outer):
     for k, v in outer.__dict__.items():
       if v is nested_class:
         return outer, k
       elif isinstance(v, (type, types.ClassType)) and hasattr(v, '__dict__'):
-        res = find_containing_class_inner(v)
+        res = _find_containing_class_inner(v)
         if res: return res
 
-  return find_containing_class_inner(sys.modules[nested_class.__module__])
+  return _find_containing_class_inner(sys.modules[nested_class.__module__])
 
 
 def _nested_type_wrapper(fun):
@@ -76,8 +76,8 @@ def wrapper(pickler, obj):
     # do anything special because the pickler itself will save the constituent
     # parts of the type (i.e., name, base classes, dictionary) and then
     # recreate it during unpickling.
-    if is_nested_class(obj) and obj.__module__ != '__main__':
-      containing_class_and_name = find_containing_class(obj)
+    if _is_nested_class(obj) and obj.__module__ != '__main__':
+      containing_class_and_name = _find_containing_class(obj)
       if containing_class_and_name is not None:
         return pickler.save_reduce(
             getattr, containing_class_and_name, obj=obj)
@@ -108,11 +108,11 @@ def wrapper(pickler, obj):
 # Dill pickles generators objects without complaint, but unpickling produces
 # TypeError: object.__new__(generator) is not safe, use generator.__new__()
 # on some versions of Python.
-def reject_generators(unused_pickler, unused_obj):
+def _reject_generators(unused_pickler, unused_obj):
   raise TypeError("can't (safely) pickle generator objects")
 
 
-dill.dill.Pickler.dispatch[types.GeneratorType] = reject_generators
+dill.dill.Pickler.dispatch[types.GeneratorType] = _reject_generators
 
 
 # This if guards against dill not being full initialized when generating docs.
@@ -185,6 +185,8 @@ def new_log_info(msg, *args, **kwargs):
 # pickler.loads() being used for data, which results in an unnecessary base64
 # encoding.  This should be cleaned up.
 def dumps(o, enable_trace=True):
+  """For internal use only; no backwards-compatibility guarantees."""
+
   try:
     s = dill.dumps(o)
   except Exception:      # pylint: disable=broad-except
@@ -206,6 +208,8 @@ def dumps(o, enable_trace=True):
 
 
 def loads(encoded, enable_trace=True):
+  """For internal use only; no backwards-compatibility guarantees."""
+
   c = base64.b64decode(encoded)
 
   s = zlib.decompress(c)
@@ -224,7 +228,9 @@ def loads(encoded, enable_trace=True):
 
 
 def dump_session(file_path):
-  """Pickle the current python session to be used in the worker.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Pickle the current python session to be used in the worker.
 
   Note: Due to the inconsistency in the first dump of dill dump_session we
   create and load the dump twice to have consistent results in the worker and
diff --git a/sdks/python/apache_beam/internal/util.py b/sdks/python/apache_beam/internal/util.py
index 5b31e88ee271..df4878c31382 100644
--- a/sdks/python/apache_beam/internal/util.py
+++ b/sdks/python/apache_beam/internal/util.py
@@ -24,7 +24,9 @@
 
 
 class ArgumentPlaceholder(object):
-  """A place holder object replacing PValues in argument lists.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A place holder object replacing PValues in argument lists.
 
   A Fn object can take any number of "side inputs", which are PValues that will
   be evaluated during pipeline execution and will be provided to the function
@@ -48,7 +50,9 @@ def __eq__(self, other):
 
 
 def remove_objects_from_args(args, kwargs, pvalue_classes):
-  """Replaces all objects of a given type in args/kwargs with a placeholder.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Replaces all objects of a given type in args/kwargs with a placeholder.
 
   Args:
     args: A list of positional arguments.
@@ -77,7 +81,9 @@ def swapper(value):
 
 
 def insert_values_in_args(args, kwargs, values):
-  """Replaces all placeholders in args/kwargs with actual values.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Replaces all placeholders in args/kwargs with actual values.
 
   Args:
     args: A list of positional arguments.
@@ -100,7 +106,9 @@ def insert_values_in_args(args, kwargs, values):
 
 
 def run_using_threadpool(fn_to_execute, inputs, pool_size):
-  """Runs the given function on given inputs using a thread pool.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Runs the given function on given inputs using a thread pool.
 
   Args:
     fn_to_execute: Function to execute
diff --git a/sdks/python/apache_beam/io/concat_source.py b/sdks/python/apache_beam/io/concat_source.py
index 1656180f0dfd..dfd16953bfa9 100644
--- a/sdks/python/apache_beam/io/concat_source.py
+++ b/sdks/python/apache_beam/io/concat_source.py
@@ -25,7 +25,9 @@
 
 
 class ConcatSource(iobase.BoundedSource):
-  """A ``BoundedSource`` that can group a set of ``BoundedSources``.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A ``BoundedSource`` that can group a set of ``BoundedSources``.
 
   Primarily for internal use, use the ``apache_beam.Flatten`` transform
   to create the union of several reads.
@@ -89,7 +91,9 @@ def default_output_coder(self):
 
 
 class ConcatRangeTracker(iobase.RangeTracker):
-  """Range tracker for ConcatSource"""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Range tracker for ConcatSource"""
 
   def __init__(self, start, end, source_bundles):
     """Initializes ``ConcatRangeTracker``
diff --git a/sdks/python/apache_beam/metrics/cells.py b/sdks/python/apache_beam/metrics/cells.py
index 41d24beda6dc..fbe3ad3efcd5 100644
--- a/sdks/python/apache_beam/metrics/cells.py
+++ b/sdks/python/apache_beam/metrics/cells.py
@@ -31,7 +31,9 @@
 
 
 class CellCommitState(object):
-  """Atomically tracks a cell's dirty/clean commit status.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Atomically tracks a cell's dirty/clean commit status.
 
   Reporting a metric update works in a two-step process: First, updates to the
   metric are received, and the metric is marked as 'dirty'. Later, updates are
@@ -102,7 +104,9 @@ def before_commit(self):
 
 
 class MetricCell(object):
-  """Accumulates in-memory changes to a metric.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Accumulates in-memory changes to a metric.
 
   A MetricCell represents a specific metric in a single context and bundle.
   All subclasses must be thread safe, as these are used in the pipeline runners,
@@ -118,7 +122,9 @@ def get_cumulative(self):
 
 
 class CounterCell(Counter, MetricCell):
-  """Tracks the current value and delta of a counter metric.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Tracks the current value and delta of a counter metric.
 
   Each cell tracks the state of a metric independently per context per bundle.
   Therefore, each metric has a different cell in each bundle, cells are
@@ -146,7 +152,9 @@ def get_cumulative(self):
 
 
 class DistributionCell(Distribution, MetricCell):
-  """Tracks the current value and delta for a distribution metric.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Tracks the current value and delta for a distribution metric.
 
   Each cell tracks the state of a metric independently per context per bundle.
   Therefore, each metric has a different cell in each bundle, that is later
@@ -228,7 +236,9 @@ def mean(self):
 
 
 class DistributionData(object):
-  """The data structure that holds data about a distribution metric.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  The data structure that holds data about a distribution metric.
 
   Distribution metrics are restricted to distributions of integers only.
 
@@ -280,7 +290,9 @@ def singleton(cls, value):
 
 
 class MetricAggregator(object):
-  """Base interface for aggregating metric data during pipeline execution."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Base interface for aggregating metric data during pipeline execution."""
   def zero(self):
     raise NotImplementedError
 
@@ -292,7 +304,9 @@ def result(self, x):
 
 
 class CounterAggregator(MetricAggregator):
-  """Aggregator for Counter metric data during pipeline execution.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Aggregator for Counter metric data during pipeline execution.
 
   Values aggregated should be ``int`` objects.
   """
@@ -307,7 +321,9 @@ def result(self, x):
 
 
 class DistributionAggregator(MetricAggregator):
-  """Aggregator for Distribution metric data during pipeline execution.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Aggregator for Distribution metric data during pipeline execution.
 
   Values aggregated should be ``DistributionData`` objects.
   """
diff --git a/sdks/python/apache_beam/metrics/execution.py b/sdks/python/apache_beam/metrics/execution.py
index dbd05330d1fd..a06ec0c2f6af 100644
--- a/sdks/python/apache_beam/metrics/execution.py
+++ b/sdks/python/apache_beam/metrics/execution.py
@@ -16,7 +16,7 @@
 #
 
 """
-Internal classes for Metrics API.
+This module is for internal use only; no backwards-compatibility guarantees.
 
 The classes in this file keep shared state, and organize metrics information.
 
@@ -36,7 +36,9 @@
 
 
 class MetricKey(object):
-  """Key used to identify instance of metric cell.
+  """
+
+  Key used to identify instance of metric cell.
 
   Metrics are internally keyed by the step name they associated with and
   the name of the metric.
@@ -193,6 +195,7 @@ def get_cumulative(self):
 
 
 class ScopedMetricsContainer(object):
+
   def __init__(self, container=None):
     self._stack = MetricsEnvironment.container_stack()
     self._container = container
diff --git a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
index 769b13cfeb7d..cb0b72b7d1fc 100644
--- a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
+++ b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
@@ -33,6 +33,8 @@
 from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 DESCRIPTOR = _descriptor.FileDescriptor(
   name='beam_fn_api.proto',
diff --git a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
index 7fbe4c10d1ba..08d7dada14a6 100644
--- a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
+++ b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
@@ -22,6 +22,8 @@
 
 import beam_fn_api_pb2 as beam__fn__api__pb2
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 class BeamFnControlStub(object):
   """
diff --git a/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py b/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
index d2006d71c9c2..e8793b63da51 100644
--- a/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
+++ b/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
@@ -33,6 +33,8 @@
 
 from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 DESCRIPTOR = _descriptor.FileDescriptor(
   name='beam_runner_api.proto',
diff --git a/sdks/python/apache_beam/runners/common.pxd b/sdks/python/apache_beam/runners/common.pxd
index 53f7aa8cc962..dcfac2e1b2bd 100644
--- a/sdks/python/apache_beam/runners/common.pxd
+++ b/sdks/python/apache_beam/runners/common.pxd
@@ -43,7 +43,7 @@ cdef class DoFnSignature(object):
 
 cdef class DoFnInvoker(object):
   cdef public DoFnSignature signature
-  cdef OutputProcessor output_processor
+  cdef _OutputProcessor output_processor
 
   cpdef invoke_process(self, WindowedValue windowed_value)
   cpdef invoke_start_bundle(self)
@@ -77,7 +77,7 @@ cdef class DoFnRunner(Receiver):
   cpdef process(self, WindowedValue windowed_value)
 
 
-cdef class OutputProcessor(object):
+cdef class _OutputProcessor(object):
   cdef object window_fn
   cdef Receiver main_receivers
   cdef object tagged_receivers
diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py
index ec1f5dc47c77..0aef0a1791a0 100644
--- a/sdks/python/apache_beam/runners/common.py
+++ b/sdks/python/apache_beam/runners/common.py
@@ -32,6 +32,7 @@
 
 
 class LoggingContext(object):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def enter(self):
     pass
@@ -41,7 +42,9 @@ def exit(self):
 
 
 class Receiver(object):
-  """An object that consumes a WindowedValue.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  An object that consumes a WindowedValue.
 
   This class can be efficiently used to pass values between the
   sdk and worker harnesses.
@@ -52,7 +55,9 @@ def receive(self, windowed_value):
 
 
 class DoFnMethodWrapper(object):
-  """Represents a method of a DoFn object."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Represents a method of a DoFn object."""
 
   def __init__(self, do_fn, method_name):
     """
@@ -299,7 +304,9 @@ def _invoke_per_window(self, windowed_value):
 
 
 class DoFnRunner(Receiver):
-  """A helper class for executing ParDo operations.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A helper class for executing ParDo operations.
   """
 
   def __init__(self,
@@ -361,7 +368,7 @@ def __init__(self,
 
     # Optimize for the common case.
     main_receivers = as_receiver(tagged_receivers[None])
-    output_processor = OutputProcessor(
+    output_processor = _OutputProcessor(
         windowing.windowfn, main_receivers, tagged_receivers)
 
     self.do_fn_invoker = DoFnInvoker.create_invoker(
@@ -411,11 +418,11 @@ def _reraise_augmented(self, exn):
       raise
 
 
-class OutputProcessor(object):
+class _OutputProcessor(object):
   """Processes output produced by DoFn method invocations."""
 
   def __init__(self, window_fn, main_receivers, tagged_receivers):
-    """Initializes ``OutputProcessor``.
+    """Initializes ``_OutputProcessor``.
 
     Args:
       window_fn: a windowing function (WindowFn).
@@ -497,7 +504,7 @@ def finish_bundle_outputs(self, results):
         self.tagged_receivers[tag].output(windowed_value)
 
 
-class NoContext(WindowFn.AssignContext):
+class _NoContext(WindowFn.AssignContext):
   """An uninspectable WindowFn.AssignContext."""
   NO_VALUE = object()
 
@@ -518,7 +525,9 @@ def existing_windows(self):
 
 
 class DoFnState(object):
-  """Keeps track of state that DoFns want, currently, user counters.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Keeps track of state that DoFns want, currently, user counters.
   """
 
   def __init__(self, counter_factory):
@@ -533,6 +542,7 @@ def counter_for(self, aggregator):
 
 # TODO(robertwb): Replace core.DoFnContext with this.
 class DoFnContext(object):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def __init__(self, label, element=None, state=None):
     self.label = label
@@ -597,6 +607,8 @@ def receive(self, windowed_value):
 
 
 def as_receiver(maybe_receiver):
+  """For internal use only; no backwards-compatibility guarantees."""
+
   if isinstance(maybe_receiver, Receiver):
     return maybe_receiver
   return _ReceiverAdapter(maybe_receiver)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 220ceb8ed325..ea4959322f1e 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -15,7 +15,9 @@
 # limitations under the License.
 #
 
-"""Dataflow client utility functions."""
+""" For internal use only. No backwards compatibility guarantees.
+
+Dataflow client utility functions."""
 
 import codecs
 import getpass
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index 63e593ce58d1..892d9f94d1ee 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -244,7 +244,9 @@ def _populate_requirements_cache(requirements_file, cache_dir):
 def stage_job_resources(
     options, file_copy=_dependency_file_copy, build_setup_args=None,
     temp_dir=None, populate_requirements_cache=_populate_requirements_cache):
-  """Creates (if needed) and stages job resources to options.staging_location.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Creates (if needed) and stages job resources to options.staging_location.
 
   Args:
     options: Command line options. More specifically the function will expect
@@ -467,7 +469,9 @@ def _stage_beam_sdk_tarball(sdk_remote_location, staged_path, temp_dir):
 
 
 def get_required_container_version():
-  """Returns the Google Cloud Dataflow container version for remote execution.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Returns the Google Cloud Dataflow container version for remote execution.
   """
   # TODO(silviuc): Handle apache-beam versions when we have official releases.
   import pkg_resources as pkg
@@ -487,7 +491,9 @@ def get_required_container_version():
 
 
 def get_sdk_name_and_version():
-  """Returns name and version of SDK reported to Google Cloud Dataflow."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Returns name and version of SDK reported to Google Cloud Dataflow."""
   # TODO(ccy): Make this check cleaner.
   container_version = get_required_container_version()
   if container_version == BEAM_CONTAINER_VERSION:
@@ -496,7 +502,9 @@ def get_sdk_name_and_version():
 
 
 def get_sdk_package_name():
-  """Returns the PyPI package name to be staged to Google Cloud Dataflow."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Returns the PyPI package name to be staged to Google Cloud Dataflow."""
   container_version = get_required_container_version()
   if container_version == BEAM_CONTAINER_VERSION:
     return BEAM_PACKAGE_NAME
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py
index 182f27e5af97..be6722427b6c 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/names.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py
@@ -29,7 +29,9 @@
 
 
 class TransformNames(object):
-  """Transform strings as they are expected in the CloudWorkflow protos."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Transform strings as they are expected in the CloudWorkflow protos."""
   COLLECTION_TO_SINGLETON = 'CollectionToSingleton'
   COMBINE = 'CombineValues'
   CREATE_PCOLLECTION = 'CreateCollection'
@@ -41,7 +43,9 @@ class TransformNames(object):
 
 
 class PropertyNames(object):
-  """Property strings as they are expected in the CloudWorkflow protos."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Property strings as they are expected in the CloudWorkflow protos."""
   BIGQUERY_CREATE_DISPOSITION = 'create_disposition'
   BIGQUERY_DATASET = 'dataset'
   BIGQUERY_QUERY = 'bigquery_query'
diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py
index 188b4af2579c..c1f4238178ac 100644
--- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py
+++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py
@@ -16,6 +16,8 @@
 #
 
 """Dataflow native sources and sinks.
+
+For internal use only; no backwards-compatibility guarantees.
 """
 
 import logging
diff --git a/sdks/python/apache_beam/runners/direct/bundle_factory.py b/sdks/python/apache_beam/runners/direct/bundle_factory.py
index 42c8095c6858..ed00b03310bb 100644
--- a/sdks/python/apache_beam/runners/direct/bundle_factory.py
+++ b/sdks/python/apache_beam/runners/direct/bundle_factory.py
@@ -24,7 +24,9 @@
 
 
 class BundleFactory(object):
-  """BundleFactory creates output bundles to be used by transform evaluators.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  BundleFactory creates output bundles to be used by transform evaluators.
 
   Args:
     stacked: whether or not to stack the WindowedValues within the bundle
@@ -36,7 +38,7 @@ def __init__(self, stacked):
     self._stacked = stacked
 
   def create_bundle(self, output_pcollection):
-    return Bundle(output_pcollection, self._stacked)
+    return _Bundle(output_pcollection, self._stacked)
 
   def create_empty_committed_bundle(self, output_pcollection):
     bundle = self.create_bundle(output_pcollection)
@@ -45,7 +47,7 @@ def create_empty_committed_bundle(self, output_pcollection):
 
 
 # a bundle represents a unit of work that will be processed by a transform.
-class Bundle(object):
+class _Bundle(object):
   """Part of a PCollection with output elements.
 
   Part of a PCollection. Elements are output to a bundle, which will cause them
@@ -67,7 +69,7 @@ class Bundle(object):
     b = Bundle(stacked=False)
   """
 
-  class StackedWindowedValues(object):
+  class _StackedWindowedValues(object):
     """A stack of WindowedValues with the same timestamp and windows.
 
     It must be initialized from a single WindowedValue.
@@ -131,7 +133,7 @@ def get_elements_iterable(self, make_copy=False):
 
     def iterable_stacked_or_elements(elements):
       for e in elements:
-        if isinstance(e, Bundle.StackedWindowedValues):
+        if isinstance(e, _Bundle._StackedWindowedValues):
           for w in e.windowed_values():
             yield w
         else:
@@ -171,11 +173,11 @@ def add(self, element):
       return
     if (self._elements and
         (isinstance(self._elements[-1], (WindowedValue,
-                                         Bundle.StackedWindowedValues))) and
+                                         _Bundle._StackedWindowedValues))) and
         self._elements[-1].timestamp == element.timestamp and
         self._elements[-1].windows == element.windows):
       if isinstance(self._elements[-1], WindowedValue):
-        self._elements[-1] = Bundle.StackedWindowedValues(self._elements[-1])
+        self._elements[-1] = _Bundle._StackedWindowedValues(self._elements[-1])
       self._elements[-1].add_value(element.value)
     else:
       self._elements.append(element)
diff --git a/sdks/python/apache_beam/runners/direct/clock.py b/sdks/python/apache_beam/runners/direct/clock.py
index dd1800a1bb85..84d52f79948b 100644
--- a/sdks/python/apache_beam/runners/direct/clock.py
+++ b/sdks/python/apache_beam/runners/direct/clock.py
@@ -23,6 +23,7 @@
 
 
 class Clock(object):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def time(self):
     """Returns the number of milliseconds since epoch."""
@@ -30,7 +31,9 @@ def time(self):
 
 
 class MockClock(Clock):
-  """Mock clock implementation for testing."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Mock clock implementation for testing."""
 
   def __init__(self, now_in_ms):
     self._now_in_ms = now_in_ms
diff --git a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor.py b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor.py
index cdfadb704cf5..d625d3ce5cee 100644
--- a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor.py
+++ b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor.py
@@ -24,7 +24,9 @@
 
 
 class ConsumerTrackingPipelineVisitor(PipelineVisitor):
-  """Visitor for extracting value-consumer relations from the graph.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Visitor for extracting value-consumer relations from the graph.
 
   Tracks the AppliedPTransforms that consume each PValue in the Pipeline. This
   is used to schedule consuming PTransforms to consume input after the upstream
diff --git a/sdks/python/apache_beam/runners/direct/executor.py b/sdks/python/apache_beam/runners/direct/executor.py
index 0aa1aec42bd7..9efbede47e9b 100644
--- a/sdks/python/apache_beam/runners/direct/executor.py
+++ b/sdks/python/apache_beam/runners/direct/executor.py
@@ -30,7 +30,7 @@
 from apache_beam.metrics.execution import ScopedMetricsContainer
 
 
-class ExecutorService(object):
+class _ExecutorService(object):
   """Thread pool for executing tasks in parallel."""
 
   class CallableTask(object):
@@ -42,14 +42,14 @@ def call(self):
     def name(self):
       return None
 
-  class ExecutorServiceWorker(threading.Thread):
+  class _ExecutorServiceWorker(threading.Thread):
     """Worker thread for executing a single task at a time."""
 
     # Amount to block waiting for getting an item from the queue in seconds.
     TIMEOUT = 5
 
     def __init__(self, queue, index):
-      super(ExecutorService.ExecutorServiceWorker, self).__init__()
+      super(_ExecutorService._ExecutorServiceWorker, self).__init__()
       self.queue = queue
       self._index = index
       self._default_name = 'ExecutorServiceWorker-' + str(index)
@@ -70,7 +70,7 @@ def _get_task_or_none(self):
         # Do not block indefinitely, otherwise we may not act for a requested
         # shutdown.
         return self.queue.get(
-            timeout=ExecutorService.ExecutorServiceWorker.TIMEOUT)
+            timeout=_ExecutorService._ExecutorServiceWorker.TIMEOUT)
       except Queue.Empty:
         return None
 
@@ -92,12 +92,12 @@ def shutdown(self):
 
   def __init__(self, num_workers):
     self.queue = Queue.Queue()
-    self.workers = [ExecutorService.ExecutorServiceWorker(
+    self.workers = [_ExecutorService._ExecutorServiceWorker(
         self.queue, i) for i in range(num_workers)]
     self.shutdown_requested = False
 
   def submit(self, task):
-    assert isinstance(task, ExecutorService.CallableTask)
+    assert isinstance(task, _ExecutorService.CallableTask)
     if not self.shutdown_requested:
       self.queue.put(task)
 
@@ -122,7 +122,7 @@ def shutdown(self):
     # last task).
 
 
-class TransformEvaluationState(object):
+class _TransformEvaluationState(object):
 
   def __init__(self, executor_service, scheduled):
     self.executor_service = executor_service
@@ -136,7 +136,7 @@ def complete(self, completed_work):
     self.scheduled.remove(completed_work)
 
 
-class ParallelEvaluationState(TransformEvaluationState):
+class _ParallelEvaluationState(_TransformEvaluationState):
   """A TransformEvaluationState with unlimited parallelism.
 
   Any TransformExecutor scheduled will be immediately submitted to the
@@ -148,7 +148,7 @@ class ParallelEvaluationState(TransformEvaluationState):
   pass
 
 
-class SerialEvaluationState(TransformEvaluationState):
+class _SerialEvaluationState(_TransformEvaluationState):
   """A TransformEvaluationState with a single work queue.
 
   Any TransformExecutor scheduled will be placed on the work queue. Only one
@@ -159,14 +159,14 @@ class SerialEvaluationState(TransformEvaluationState):
   """
 
   def __init__(self, executor_service, scheduled):
-    super(SerialEvaluationState, self).__init__(executor_service, scheduled)
+    super(_SerialEvaluationState, self).__init__(executor_service, scheduled)
     self.serial_queue = collections.deque()
     self.currently_evaluating = None
     self._lock = threading.Lock()
 
   def complete(self, completed_work):
     self._update_currently_evaluating(None, completed_work)
-    super(SerialEvaluationState, self).complete(completed_work)
+    super(_SerialEvaluationState, self).complete(completed_work)
 
   def schedule(self, new_work):
     self._update_currently_evaluating(new_work, None)
@@ -181,10 +181,10 @@ def _update_currently_evaluating(self, new_work, completed_work):
       if self.serial_queue and not self.currently_evaluating:
         next_work = self.serial_queue.pop()
         self.currently_evaluating = next_work
-        super(SerialEvaluationState, self).schedule(next_work)
+        super(_SerialEvaluationState, self).schedule(next_work)
 
 
-class TransformExecutorServices(object):
+class _TransformExecutorServices(object):
   """Schedules and completes TransformExecutors.
 
   Controls the concurrency as appropriate for the applied transform the executor
@@ -194,7 +194,7 @@ class TransformExecutorServices(object):
   def __init__(self, executor_service):
     self._executor_service = executor_service
     self._scheduled = set()
-    self._parallel = ParallelEvaluationState(
+    self._parallel = _ParallelEvaluationState(
         self._executor_service, self._scheduled)
     self._serial_cache = WeakValueDictionary()
 
@@ -204,7 +204,7 @@ def parallel(self):
   def serial(self, step):
     cached = self._serial_cache.get(step)
     if not cached:
-      cached = SerialEvaluationState(self._executor_service, self._scheduled)
+      cached = _SerialEvaluationState(self._executor_service, self._scheduled)
       self._serial_cache[step] = cached
     return cached
 
@@ -230,17 +230,19 @@ def handle_result(self, input_committed_bundle, transform_result):
     output_committed_bundles = self._evaluation_context.handle_result(
         input_committed_bundle, self._timers, transform_result)
     for output_committed_bundle in output_committed_bundles:
-      self._all_updates.offer(_ExecutorServiceParallelExecutor.ExecutorUpdate(
+      self._all_updates.offer(_ExecutorServiceParallelExecutor._ExecutorUpdate(
           output_committed_bundle, None))
     return output_committed_bundles
 
   def handle_exception(self, exception):
     self._all_updates.offer(
-        _ExecutorServiceParallelExecutor.ExecutorUpdate(None, exception))
+        _ExecutorServiceParallelExecutor._ExecutorUpdate(None, exception))
 
 
-class TransformExecutor(ExecutorService.CallableTask):
-  """TransformExecutor will evaluate a bundle using an applied ptransform.
+class TransformExecutor(_ExecutorService.CallableTask):
+  """For internal use only; no backwards-compatibility guarantees.
+
+  TransformExecutor will evaluate a bundle using an applied ptransform.
 
   A CallableTask responsible for constructing a TransformEvaluator and
   evaluating it on some bundle of input, and registering the result using the
@@ -316,6 +318,7 @@ def call(self):
 
 
 class Executor(object):
+  """For internal use only; no backwards-compatibility guarantees."""
 
   def __init__(self, *args, **kwargs):
     self._executor = _ExecutorServiceParallelExecutor(*args, **kwargs)
@@ -334,17 +337,17 @@ class _ExecutorServiceParallelExecutor(object):
 
   def __init__(self, value_to_consumers, transform_evaluator_registry,
                evaluation_context):
-    self.executor_service = ExecutorService(
+    self.executor_service = _ExecutorService(
         _ExecutorServiceParallelExecutor.NUM_WORKERS)
-    self.transform_executor_services = TransformExecutorServices(
+    self.transform_executor_services = _TransformExecutorServices(
         self.executor_service)
     self.value_to_consumers = value_to_consumers
     self.transform_evaluator_registry = transform_evaluator_registry
     self.evaluation_context = evaluation_context
     self.all_updates = _ExecutorServiceParallelExecutor._TypedUpdateQueue(
-        _ExecutorServiceParallelExecutor.ExecutorUpdate)
+        _ExecutorServiceParallelExecutor._ExecutorUpdate)
     self.visible_updates = _ExecutorServiceParallelExecutor._TypedUpdateQueue(
-        _ExecutorServiceParallelExecutor.VisibleExecutorUpdate)
+        _ExecutorServiceParallelExecutor._VisibleExecutorUpdate)
     self.default_completion_callback = _CompletionCallback(
         evaluation_context, self.all_updates)
 
@@ -412,7 +415,7 @@ def offer(self, item):
       assert isinstance(item, self._item_type)
       self._queue.put_nowait(item)
 
-  class ExecutorUpdate(object):
+  class _ExecutorUpdate(object):
     """An internal status update on the state of the executor."""
 
     def __init__(self, produced_bundle=None, exception=None):
@@ -425,7 +428,7 @@ def __init__(self, produced_bundle=None, exception=None):
         # Not the right exception.
         self.exc_info = (exception, None, None)
 
-  class VisibleExecutorUpdate(object):
+  class _VisibleExecutorUpdate(object):
     """An update of interest to the user.
 
     Used for awaiting the completion to decide whether to return normally or
@@ -437,7 +440,7 @@ def __init__(self, exc_info=(None, None, None)):
       self.exception = exc_info[1] or exc_info[0]
       self.exc_info = exc_info
 
-  class _MonitorTask(ExecutorService.CallableTask):
+  class _MonitorTask(_ExecutorService.CallableTask):
     """MonitorTask continuously runs to ensure that pipeline makes progress."""
 
     def __init__(self, executor):
@@ -458,7 +461,7 @@ def call(self):
             logging.warning('A task failed with exception.\n %s',
                             update.exception)
             self._executor.visible_updates.offer(
-                _ExecutorServiceParallelExecutor.VisibleExecutorUpdate(
+                _ExecutorServiceParallelExecutor._VisibleExecutorUpdate(
                     update.exc_info))
           update = self._executor.all_updates.poll()
         self._executor.evaluation_context.schedule_pending_unblocked_tasks(
@@ -467,7 +470,7 @@ def call(self):
       except Exception as e:  # pylint: disable=broad-except
         logging.error('Monitor task died due to exception.\n %s', e)
         self._executor.visible_updates.offer(
-            _ExecutorServiceParallelExecutor.VisibleExecutorUpdate(
+            _ExecutorServiceParallelExecutor._VisibleExecutorUpdate(
                 sys.exc_info()))
       finally:
         if not self._should_shutdown():
@@ -497,11 +500,11 @@ def _should_shutdown(self):
       else:
         if self._executor.evaluation_context.is_done():
           self._executor.visible_updates.offer(
-              _ExecutorServiceParallelExecutor.VisibleExecutorUpdate())
+              _ExecutorServiceParallelExecutor._VisibleExecutorUpdate())
         else:
           # Nothing is scheduled for execution, but watermarks incomplete.
           self._executor.visible_updates.offer(
-              _ExecutorServiceParallelExecutor.VisibleExecutorUpdate(
+              _ExecutorServiceParallelExecutor._VisibleExecutorUpdate(
                   (Exception('Monitor task detected a pipeline stall.'),
                    None,
                    None)))
diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
index f6cdd414eb48..6984ded7cc3f 100644
--- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py
+++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
@@ -41,7 +41,9 @@
 
 
 class TransformEvaluatorRegistry(object):
-  """Creates instances of TransformEvaluator for the application of a transform.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Creates instances of TransformEvaluator for the application of a transform.
   """
 
   def __init__(self, evaluation_context):
@@ -253,7 +255,7 @@ class NullReceiver(object):
     def output(self, element):
       pass
 
-  class InMemoryReceiver(object):
+  class _InMemoryReceiver(object):
     """Buffers undeclared outputs to the given dictionary."""
 
     def __init__(self, target, tag):
@@ -267,7 +269,7 @@ def __missing__(self, key):
     if self._evaluation_context.has_cache:
       if not self._undeclared_in_memory_tag_values:
         self._undeclared_in_memory_tag_values = collections.defaultdict(list)
-      receiver = _TaggedReceivers.InMemoryReceiver(
+      receiver = _TaggedReceivers._InMemoryReceiver(
           self._undeclared_in_memory_tag_values, key)
     else:
       if not self._null_receiver:
diff --git a/sdks/python/apache_beam/runners/direct/transform_result.py b/sdks/python/apache_beam/runners/direct/transform_result.py
index 8ae0aeab24b2..febdd202aa0a 100644
--- a/sdks/python/apache_beam/runners/direct/transform_result.py
+++ b/sdks/python/apache_beam/runners/direct/transform_result.py
@@ -21,7 +21,9 @@
 
 
 class TransformResult(object):
-  """The result of evaluating an AppliedPTransform with a TransformEvaluator."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  The result of evaluating an AppliedPTransform with a TransformEvaluator."""
 
   def __init__(self, applied_ptransform, uncommitted_output_bundles, state,
                timer_update, counters, watermark_hold,
diff --git a/sdks/python/apache_beam/runners/direct/watermark_manager.py b/sdks/python/apache_beam/runners/direct/watermark_manager.py
index 7793478a85a8..3a135397e12f 100644
--- a/sdks/python/apache_beam/runners/direct/watermark_manager.py
+++ b/sdks/python/apache_beam/runners/direct/watermark_manager.py
@@ -28,7 +28,9 @@
 
 
 class WatermarkManager(object):
-  """Tracks and updates watermarks for all AppliedPTransforms."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Tracks and updates watermarks for all AppliedPTransforms."""
 
   WATERMARK_POS_INF = MAX_TIMESTAMP
   WATERMARK_NEG_INF = MIN_TIMESTAMP
@@ -41,12 +43,12 @@ def __init__(self, clock, root_transforms, value_to_consumers):
     self._transform_to_watermarks = {}
 
     for root_transform in root_transforms:
-      self._transform_to_watermarks[root_transform] = TransformWatermarks(
+      self._transform_to_watermarks[root_transform] = _TransformWatermarks(
           self._clock)
 
     for consumers in value_to_consumers.values():
       for consumer in consumers:
-        self._transform_to_watermarks[consumer] = TransformWatermarks(
+        self._transform_to_watermarks[consumer] = _TransformWatermarks(
             self._clock)
 
     for consumers in value_to_consumers.values():
@@ -139,7 +141,7 @@ def extract_fired_timers(self):
     return all_timers
 
 
-class TransformWatermarks(object):
+class _TransformWatermarks(object):
   """Tracks input and output watermarks for aan AppliedPTransform."""
 
   def __init__(self, clock):
diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py
index 4f827749d0a4..d3d3c2468321 100644
--- a/sdks/python/apache_beam/runners/pipeline_context.py
+++ b/sdks/python/apache_beam/runners/pipeline_context.py
@@ -60,7 +60,9 @@ def get_by_id(self, id):
 
 
 class PipelineContext(object):
-  """Used for accessing and constructing the referenced objects of a Pipeline.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Used for accessing and constructing the referenced objects of a Pipeline.
   """
 
   _COMPONENT_TYPES = {
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
index 5802c17f4276..26355595e991 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
@@ -39,6 +39,8 @@
 from apache_beam.runners.worker import operation_specs
 from apache_beam.runners.worker import sdk_worker
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 def streaming_rpc_handler(cls, method_name):
   """Un-inverts the flow of control between the runner and the sdk harness."""
diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
index 3e0878002581..077871ed50ca 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
@@ -44,6 +44,8 @@
 from apache_beam.utils import profiler
 from apache_beam.utils.counters import CounterFactory
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 class MapTaskExecutorRunner(PipelineRunner):
   """Beam runner translating a pipeline into map tasks that are then executed.
diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py
index b35cb7b77f20..d875fdc11a45 100644
--- a/sdks/python/apache_beam/runners/runner.py
+++ b/sdks/python/apache_beam/runners/runner.py
@@ -56,7 +56,9 @@ def _get_runner_map(runner_names, module_path):
 
 
 def create_runner(runner_name):
-  """Creates a runner instance from a runner class name.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Creates a runner instance from a runner class name.
 
   Args:
     runner_name: Name of the pipeline runner. Possible values are:
@@ -170,7 +172,9 @@ def run_transform(self, transform_node):
 
 
 class PValueCache(object):
-  """Local cache for arbitrary information computed for PValue objects."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Local cache for arbitrary information computed for PValue objects."""
 
   def __init__(self, use_disk_backed_cache=False):
     # Cache of values computed while a runner executes a pipeline. This is a
diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py
index 6425447994f6..5edd0b490750 100644
--- a/sdks/python/apache_beam/runners/worker/data_plane.py
+++ b/sdks/python/apache_beam/runners/worker/data_plane.py
@@ -31,6 +31,8 @@
 from apache_beam.runners.api import beam_fn_api_pb2
 import grpc
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 class ClosableOutputStream(type(coder_impl.create_OutputStream())):
   """A Outputstream for use with CoderImpls that has a close() method."""
diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py
index b9e36ad2c312..59ffbf4f45a2 100644
--- a/sdks/python/apache_beam/runners/worker/log_handler.py
+++ b/sdks/python/apache_beam/runners/worker/log_handler.py
@@ -24,6 +24,8 @@
 from apache_beam.runners.api import beam_fn_api_pb2
 import grpc
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 class FnApiLogRecordHandler(logging.Handler):
   """A handler that writes log records to the fn API."""
diff --git a/sdks/python/apache_beam/runners/worker/logger.py b/sdks/python/apache_beam/runners/worker/logger.py
index 217dc5835075..043353807a31 100644
--- a/sdks/python/apache_beam/runners/worker/logger.py
+++ b/sdks/python/apache_beam/runners/worker/logger.py
@@ -24,6 +24,8 @@
 
 from apache_beam.runners.common import LoggingContext
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 # Per-thread worker information. This is used only for logging to set
 # context information that changes while work items get executed:
diff --git a/sdks/python/apache_beam/runners/worker/opcounters.py b/sdks/python/apache_beam/runners/worker/opcounters.py
index 56ce0db6c8be..2bb15fa7ee4b 100644
--- a/sdks/python/apache_beam/runners/worker/opcounters.py
+++ b/sdks/python/apache_beam/runners/worker/opcounters.py
@@ -25,6 +25,8 @@
 
 from apache_beam.utils.counters import Counter
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 class SumAccumulator(object):
   """Accumulator for collecting byte counts."""
diff --git a/sdks/python/apache_beam/runners/worker/operation_specs.py b/sdks/python/apache_beam/runners/worker/operation_specs.py
index 977e165afe7c..c03d9a25ec5f 100644
--- a/sdks/python/apache_beam/runners/worker/operation_specs.py
+++ b/sdks/python/apache_beam/runners/worker/operation_specs.py
@@ -25,6 +25,8 @@
 
 from apache_beam import coders
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 def build_worker_instruction(*args):
   """Create an object representing a ParallelInstruction protobuf.
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py
index 6907f6edc92d..596bb907b582 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py
@@ -40,6 +40,10 @@
 from apache_beam.runners.api import beam_fn_api_pb2
 from apache_beam.runners.worker import operation_specs
 from apache_beam.runners.worker import operations
+
+# This module is experimental. No backwards-compatibility guarantees.
+
+
 try:
   from apache_beam.runners.worker import statesampler
 except ImportError:
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
index 28828c3b4925..b8917791b91e 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
@@ -28,6 +28,8 @@
 from apache_beam.runners.worker.log_handler import FnApiLogRecordHandler
 from apache_beam.runners.worker.sdk_worker import SdkHarness
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 def main(unused_argv):
   """Main entry point for SDK Fn Harness."""
diff --git a/sdks/python/apache_beam/runners/worker/sideinputs.py b/sdks/python/apache_beam/runners/worker/sideinputs.py
index 3bac3d9a2e4b..bdf9f4e71f5e 100644
--- a/sdks/python/apache_beam/runners/worker/sideinputs.py
+++ b/sdks/python/apache_beam/runners/worker/sideinputs.py
@@ -26,6 +26,8 @@
 from apache_beam.io import iobase
 from apache_beam.transforms import window
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 # Maximum number of reader threads for reading side input sources, per side
 # input.
diff --git a/sdks/python/apache_beam/runners/worker/statesampler_fake.py b/sdks/python/apache_beam/runners/worker/statesampler_fake.py
index efd7f2db8fcd..88ace8c5ae89 100644
--- a/sdks/python/apache_beam/runners/worker/statesampler_fake.py
+++ b/sdks/python/apache_beam/runners/worker/statesampler_fake.py
@@ -15,6 +15,8 @@
 # limitations under the License.
 #
 
+# This module is experimental. No backwards-compatibility guarantees.
+
 
 class StateSampler(object):
 
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index e37a387e6d18..d42115c5c23f 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -227,7 +227,9 @@ def _fn_takes_side_inputs(fn):
 
 
 class CallableWrapperDoFn(DoFn):
-  """A DoFn (function) object wrapping a callable object.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A DoFn (function) object wrapping a callable object.
 
   The purpose of this class is to conveniently wrap simple functions and use
   them in transforms.
@@ -410,7 +412,9 @@ def maybe_from_callable(fn):
 
 
 class CallableWrapperCombineFn(CombineFn):
-  """A CombineFn (function) object wrapping a callable object.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A CombineFn (function) object wrapping a callable object.
 
   The purpose of this class is to conveniently wrap simple functions and use
   them in Combine transforms.
@@ -537,7 +541,9 @@ def partition_for(self, element, num_partitions, *args, **kwargs):
 
 
 class CallableWrapperPartitionFn(PartitionFn):
-  """A PartitionFn object wrapping a callable object.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A PartitionFn object wrapping a callable object.
 
   Instances of this class wrap simple functions for use in Partition operations.
   """
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index 526291c43fc9..fb79b19be982 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -125,7 +125,7 @@ def visit(self, node, pvalues=None):
       super(GetPValues, self).visit(node, pvalues)
 
 
-class ZipPValues(_PValueishTransform):
+class _ZipPValues(_PValueishTransform):
   """Pairs each PValue in a pvalueish with a value in a parallel out sibling.
 
   Sibling should have the same nested structure as pvalueish.  Leaves in
@@ -148,7 +148,7 @@ def visit(self, pvalueish, sibling, pairs=None, context=None):
     elif isinstance(pvalueish, (pvalue.PValue, pvalue.DoOutputsTuple)):
       pairs.append((context, pvalueish, sibling))
     else:
-      super(ZipPValues, self).visit(pvalueish, sibling, pairs, context)
+      super(_ZipPValues, self).visit(pvalueish, sibling, pairs, context)
 
   def visit_list(self, pvalueish, sibling, pairs, context):
     if isinstance(sibling, (list, tuple)):
@@ -264,7 +264,7 @@ def type_check_inputs_or_outputs(self, pvalueish, input_or_output):
               self.__class__, input_or_output))
     root_hint = (
         arg_hints[0] if len(arg_hints) == 1 else arg_hints or kwarg_hints)
-    for context, pvalue_, hint in ZipPValues().visit(pvalueish, root_hint):
+    for context, pvalue_, hint in _ZipPValues().visit(pvalueish, root_hint):
       if pvalue_.element_type is None:
         # TODO(robertwb): It's a bug that we ever get here. (typecheck)
         continue
@@ -341,7 +341,7 @@ def __rrshift__(self, label):
   def __or__(self, right):
     """Used to compose PTransforms, e.g., ptransform1 | ptransform2."""
     if isinstance(right, PTransform):
-      return ChainedPTransform(self, right)
+      return _ChainedPTransform(self, right)
     return NotImplemented
 
   def __ror__(self, left, label=None):
@@ -453,10 +453,10 @@ def from_runner_api_parameter(spec_parameter, unused_context):
     PTransform.from_runner_api_parameter)
 
 
-class ChainedPTransform(PTransform):
+class _ChainedPTransform(PTransform):
 
   def __init__(self, *parts):
-    super(ChainedPTransform, self).__init__(label=self._chain_label(parts))
+    super(_ChainedPTransform, self).__init__(label=self._chain_label(parts))
     self._parts = parts
 
   def _chain_label(self, parts):
@@ -466,7 +466,7 @@ def __or__(self, right):
     if isinstance(right, PTransform):
       # Create a flat list rather than a nested tree of composite
       # transforms for better monitoring, etc.
-      return ChainedPTransform(*(self._parts + (right,)))
+      return _ChainedPTransform(*(self._parts + (right,)))
     return NotImplemented
 
   def expand(self, pval):
diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py
index 97240d392d2c..2cb7ce32702d 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -47,7 +47,7 @@ class AccumulationMode(object):
   # RETRACTING = 3
 
 
-class StateTag(object):
+class _StateTag(object):
   """An identifier used to store and retrieve typed, combinable state.
 
   The given tag must be unique for this stage.  If CombineFn is None then
@@ -60,22 +60,22 @@ def __init__(self, tag):
     self.tag = tag
 
 
-class ValueStateTag(StateTag):
+class _ValueStateTag(_StateTag):
   """StateTag pointing to an element."""
 
   def __repr__(self):
     return 'ValueStateTag(%s)' % (self.tag)
 
   def with_prefix(self, prefix):
-    return ValueStateTag(prefix + self.tag)
+    return _ValueStateTag(prefix + self.tag)
 
 
-class CombiningValueStateTag(StateTag):
+class _CombiningValueStateTag(_StateTag):
   """StateTag pointing to an element, accumulated with a combiner."""
 
   # TODO(robertwb): Also store the coder (perhaps extracted from the combine_fn)
   def __init__(self, tag, combine_fn):
-    super(CombiningValueStateTag, self).__init__(tag)
+    super(_CombiningValueStateTag, self).__init__(tag)
     if not combine_fn:
       raise ValueError('combine_fn must be specified.')
     if not isinstance(combine_fn, core.CombineFn):
@@ -86,22 +86,22 @@ def __repr__(self):
     return 'CombiningValueStateTag(%s, %s)' % (self.tag, self.combine_fn)
 
   def with_prefix(self, prefix):
-    return CombiningValueStateTag(prefix + self.tag, self.combine_fn)
+    return _CombiningValueStateTag(prefix + self.tag, self.combine_fn)
 
 
-class ListStateTag(StateTag):
+class _ListStateTag(_StateTag):
   """StateTag pointing to a list of elements."""
   def __repr__(self):
     return 'ListStateTag(%s)' % self.tag
 
   def with_prefix(self, prefix):
-    return ListStateTag(prefix + self.tag)
+    return _ListStateTag(prefix + self.tag)
 
 
-class WatermarkHoldStateTag(StateTag):
+class _WatermarkHoldStateTag(_StateTag):
 
   def __init__(self, tag, timestamp_combiner_impl):
-    super(WatermarkHoldStateTag, self).__init__(tag)
+    super(_WatermarkHoldStateTag, self).__init__(tag)
     self.timestamp_combiner_impl = timestamp_combiner_impl
 
   def __repr__(self):
@@ -109,8 +109,8 @@ def __repr__(self):
                                               self.timestamp_combiner_impl)
 
   def with_prefix(self, prefix):
-    return WatermarkHoldStateTag(prefix + self.tag,
-                                 self.timestamp_combiner_impl)
+    return _WatermarkHoldStateTag(prefix + self.tag,
+                                  self.timestamp_combiner_impl)
 
 
 # pylint: disable=unused-argument
@@ -251,7 +251,7 @@ class AfterWatermark(TriggerFn):
       late: if not None, a speculative trigger to repeatedly evaluate after
         the watermark passes the end of the window
   """
-  LATE_TAG = CombiningValueStateTag('is_late', any)
+  LATE_TAG = _CombiningValueStateTag('is_late', any)
 
   def __init__(self, early=None, late=None):
     self.early = Repeatedly(early) if early else None
@@ -355,7 +355,7 @@ def to_runner_api(self, context):
 class AfterCount(TriggerFn):
   """Fire when there are at least count elements in this window pane."""
 
-  COUNT_TAG = CombiningValueStateTag('count', combiners.CountCombineFn())
+  COUNT_TAG = _CombiningValueStateTag('count', combiners.CountCombineFn())
 
   def __init__(self, count):
     self.count = count
@@ -432,7 +432,7 @@ def to_runner_api(self, context):
             subtrigger=self.underlying.to_runner_api(context)))
 
 
-class ParallelTriggerFn(TriggerFn):
+class _ParallelTriggerFn(TriggerFn):
 
   __metaclass__ = ABCMeta
 
@@ -506,7 +506,7 @@ def to_runner_api(self, context):
       raise NotImplementedError(self)
 
 
-class AfterAny(ParallelTriggerFn):
+class AfterAny(_ParallelTriggerFn):
   """Fires when any subtrigger fires.
 
   Also finishes when any subtrigger finishes.
@@ -514,7 +514,7 @@ class AfterAny(ParallelTriggerFn):
   combine_op = any
 
 
-class AfterAll(ParallelTriggerFn):
+class AfterAll(_ParallelTriggerFn):
   """Fires when all subtriggers have fired.
 
   Also finishes when all subtriggers have finished.
@@ -524,7 +524,7 @@ class AfterAll(ParallelTriggerFn):
 
 class AfterEach(TriggerFn):
 
-  INDEX_TAG = CombiningValueStateTag('index', (
+  INDEX_TAG = _CombiningValueStateTag('index', (
       lambda indices: 0 if not indices else max(indices)))
 
   def __init__(self, *triggers):
@@ -711,7 +711,7 @@ class MergeableStateAdapter(SimpleState):
   # TODO(robertwb): A similar indirection could be used for sliding windows
   # or other window_fns when a single element typically belongs to many windows.
 
-  WINDOW_IDS = ValueStateTag('window_ids')
+  WINDOW_IDS = _ValueStateTag('window_ids')
 
   def __init__(self, raw_state):
     self.raw_state = raw_state
@@ -726,7 +726,7 @@ def clear_timer(self, window, name, time_domain):
       self.raw_state.clear_timer(window_id, name, time_domain)
 
   def add_state(self, window, tag, value):
-    if isinstance(tag, ValueStateTag):
+    if isinstance(tag, _ValueStateTag):
       raise ValueError(
           'Merging requested for non-mergeable state tag: %r.' % tag)
     self.raw_state.add_state(self._get_id(window), tag, value)
@@ -734,10 +734,10 @@ def add_state(self, window, tag, value):
   def get_state(self, window, tag):
     values = [self.raw_state.get_state(window_id, tag)
               for window_id in self._get_ids(window)]
-    if isinstance(tag, ValueStateTag):
+    if isinstance(tag, _ValueStateTag):
       raise ValueError(
           'Merging requested for non-mergeable state tag: %r.' % tag)
-    elif isinstance(tag, CombiningValueStateTag):
+    elif isinstance(tag, _CombiningValueStateTag):
       # TODO(robertwb): Strip combine_fn.extract_output from raw_state tag.
       if not values:
         accumulator = tag.combine_fn.create_accumulator()
@@ -747,9 +747,9 @@ def get_state(self, window, tag):
         accumulator = tag.combine_fn.merge_accumulators(values)
         # TODO(robertwb): Store the merged value in the first tag.
       return tag.combine_fn.extract_output(accumulator)
-    elif isinstance(tag, ListStateTag):
+    elif isinstance(tag, _ListStateTag):
       return [v for vs in values for v in vs]
-    elif isinstance(tag, WatermarkHoldStateTag):
+    elif isinstance(tag, _WatermarkHoldStateTag):
       return tag.timestamp_combiner_impl.combine_all(values)
     else:
       raise ValueError('Invalid tag.', tag)
@@ -904,15 +904,15 @@ class GeneralTriggerDriver(TriggerDriver):
 
   Suitable for all variants of Windowing.
   """
-  ELEMENTS = ListStateTag('elements')
-  TOMBSTONE = CombiningValueStateTag('tombstone', combiners.CountCombineFn())
+  ELEMENTS = _ListStateTag('elements')
+  TOMBSTONE = _CombiningValueStateTag('tombstone', combiners.CountCombineFn())
 
   def __init__(self, windowing):
     self.window_fn = windowing.windowfn
     self.timestamp_combiner_impl = TimestampCombiner.get_impl(
         windowing.timestamp_combiner, self.window_fn)
     # pylint: disable=invalid-name
-    self.WATERMARK_HOLD = WatermarkHoldStateTag(
+    self.WATERMARK_HOLD = _WatermarkHoldStateTag(
         'watermark', self.timestamp_combiner_impl)
     # pylint: enable=invalid-name
     self.trigger_fn = windowing.triggerfn
@@ -1035,7 +1035,7 @@ def __init__(self, defensive_copy=True):
     self.defensive_copy = defensive_copy
 
   def set_global_state(self, tag, value):
-    assert isinstance(tag, ValueStateTag)
+    assert isinstance(tag, _ValueStateTag)
     if self.defensive_copy:
       value = copy.deepcopy(value)
     self.global_state[tag.tag] = value
@@ -1055,26 +1055,26 @@ def get_window(self, window_id):
   def add_state(self, window, tag, value):
     if self.defensive_copy:
       value = copy.deepcopy(value)
-    if isinstance(tag, ValueStateTag):
+    if isinstance(tag, _ValueStateTag):
       self.state[window][tag.tag] = value
-    elif isinstance(tag, CombiningValueStateTag):
+    elif isinstance(tag, _CombiningValueStateTag):
       self.state[window][tag.tag].append(value)
-    elif isinstance(tag, ListStateTag):
+    elif isinstance(tag, _ListStateTag):
       self.state[window][tag.tag].append(value)
-    elif isinstance(tag, WatermarkHoldStateTag):
+    elif isinstance(tag, _WatermarkHoldStateTag):
       self.state[window][tag.tag].append(value)
     else:
       raise ValueError('Invalid tag.', tag)
 
   def get_state(self, window, tag):
     values = self.state[window][tag.tag]
-    if isinstance(tag, ValueStateTag):
+    if isinstance(tag, _ValueStateTag):
       return values
-    elif isinstance(tag, CombiningValueStateTag):
+    elif isinstance(tag, _CombiningValueStateTag):
       return tag.combine_fn.apply(values)
-    elif isinstance(tag, ListStateTag):
+    elif isinstance(tag, _ListStateTag):
       return values
-    elif isinstance(tag, WatermarkHoldStateTag):
+    elif isinstance(tag, _WatermarkHoldStateTag):
       return tag.timestamp_combiner_impl.combine_all(values)
     else:
       raise ValueError('Invalid tag.', tag)

From 77712ea2a1ad5685d587c5a8ae59a547a6930ae4 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Tue, 9 May 2017 18:21:39 -0700
Subject: [PATCH 166/387] Remove Readme files.

extensions -> moving to website @
https://github.com/apache/beam-site/pull/237

jdbc/test -> obsolete content, removed.
---
 sdks/java/extensions/join-library/README.md | 42 ---------------------
 sdks/java/extensions/sorter/README.md       | 42 ---------------------
 sdks/java/io/jdbc/src/test/README.md        | 32 ----------------
 3 files changed, 116 deletions(-)
 delete mode 100644 sdks/java/extensions/join-library/README.md
 delete mode 100644 sdks/java/extensions/sorter/README.md
 delete mode 100644 sdks/java/io/jdbc/src/test/README.md

diff --git a/sdks/java/extensions/join-library/README.md b/sdks/java/extensions/join-library/README.md
deleted file mode 100644
index feee64fe571d..000000000000
--- a/sdks/java/extensions/join-library/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-Join-library
-============
-
-Join-library provides inner join, outer left and right join functions to
-Apache Beam. The aim is to simplify the most common cases of join to a
-simple function call.
-
-The functions are generic so it supports join of any types supported by
-Beam. Input to the join functions are PCollections of Key/Values. Both the
-left and right PCollections need the same type for the key. All the join
-functions return a Key/Value where Key is the join key and value is
-a Key/Value where the key is the left value and right is the value.
-
-In the cases of outer join, since null cannot be serialized the user have
-to provide a value that represent null for that particular use case.
-
-Example how to use join-library:
-
-    PCollection<KV<String, String>> leftPcollection = ...
-    PCollection<KV<String, Long>> rightPcollection = ...
-
-    PCollection<KV<String, KV<String, Long>>> joinedPcollection =
-      Join.innerJoin(leftPcollection, rightPcollection);
diff --git a/sdks/java/extensions/sorter/README.md b/sdks/java/extensions/sorter/README.md
deleted file mode 100644
index 6ff3dbeb2f87..000000000000
--- a/sdks/java/extensions/sorter/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-#Sorter
-This module provides the SortValues transform, which takes a `PCollection<KV<K, Iterable<KV<K2, V>>>>` and produces a `PCollection<KV<K, Iterable<KV<K2, V>>>>` where, for each primary key `K` the paired `Iterable<KV<K2, V>>` has been sorted by the byte encoding of secondary key (`K2`). It will efficiently and scalably sort the iterables, even if they are large (do not fit in memory).
-
-##Caveats
-* This transform performs value-only sorting; the iterable accompanying each key is sorted, but *there is no relationship between different keys*, as Beam does not support any defined relationship between different elements in a PCollection.
-* Each `Iterable<KV<K2, V>>` is sorted on a single worker using local memory and disk. This means that `SortValues` may be a performance and/or scalability bottleneck when used in different pipelines. For example, users are discouraged from using `SortValues` on a `PCollection` of a single element to globally sort a large `PCollection`. A (rough) estimate of the number of bytes of disk space utilized if sorting spills to disk is `numRecords * (numSecondaryKeyBytesPerRecord + numValueBytesPerRecord + 16) * 3`.
-
-##Options
-* The user can customize the temporary location used if sorting requires spilling to disk and the maximum amount of memory to use by creating a custom instance of `BufferedExternalSorter.Options` to pass into `SortValues.create`.
-
-##Using `SortValues`
-```java
-PCollection<KV<String, KV<String, Integer>>> input = ...
-
-// Group by primary key, bringing <SecondaryKey, Value> pairs for the same key together.
-PCollection<KV<String, Iterable<KV<String, Integer>>>> grouped =
-    input.apply(GroupByKey.<String, KV<String, Integer>>create());
-
-// For every primary key, sort the iterable of <SecondaryKey, Value> pairs by secondary key.
-PCollection<KV<String, Iterable<KV<String, Integer>>>> groupedAndSorted =
-    grouped.apply(
-        SortValues.<String, String, Integer>create(new BufferedExternalSorter.Options()));
-```
diff --git a/sdks/java/io/jdbc/src/test/README.md b/sdks/java/io/jdbc/src/test/README.md
deleted file mode 100644
index 5a7ac99bedba..000000000000
--- a/sdks/java/io/jdbc/src/test/README.md
+++ /dev/null
@@ -1,32 +0,0 @@
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
--->
-
-These are instructions for maintaining postgres as needed for Integration Tests (JdbcIOIT).
-
-You can always ignore these instructions if you have your own postgres cluster to test against.
-
-Setting up Postgres
--------------------
-1. Setup kubectl so it is configured to work with your kubernetes cluster
-1. Run the postgres setup script
-    src/test/resources/kubernetes/setup.sh
-1. Do the data loading - create the data store instance by following the instructions in JdbcTestDataSet
-
-... and your postgres instances are set up!
-

From c755921f42cb460ba2cf4933cc1f771eb971f6bc Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 9 May 2017 20:46:43 -0700
Subject: [PATCH 167/387] Remove Timer.cancel() from user-facing API

---
 .../java/org/apache/beam/runners/core/SimpleDoFnRunner.java | 5 -----
 .../core/src/main/java/org/apache/beam/sdk/state/Timer.java | 6 ------
 2 files changed, 11 deletions(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index aab34a51f5eb..85423c0431b4 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -988,11 +988,6 @@ private void setUnderlyingTimer(Instant target) {
       timerInternals.setTimer(namespace, timerId, target, spec.getTimeDomain());
     }
 
-    @Override
-    public void cancel() {
-      timerInternals.deleteTimer(namespace, timerId);
-    }
-
     private Instant getCurrentTime() {
       switch(spec.getTimeDomain()) {
         case EVENT_TIME:
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
index 6f98ff82ee2e..43196a49fe70 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timer.java
@@ -53,12 +53,6 @@ public interface Timer {
    */
   void set(Instant absoluteTime);
 
-  /**
-   * Unsets this timer. It is permitted to {@code cancel()} whether or not the timer was actually
-   * set.
-   */
-  void cancel();
-
   /**
    * Sets the timer relative to the current time, according to any offset and alignment specified.
    * Using {@link #offset(Duration)} and {@link #align(Duration)}.

From af525e0cf52f60b98326388c4a6bb220b1f9ff3a Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Tue, 9 May 2017 16:38:50 -0700
Subject: [PATCH 168/387] [BEAM-2211] Delete deprecated NoopPathValidator

---
 .../runners/dataflow/DataflowMetricsTest.java |  2 +-
 .../dataflow/DataflowPipelineJobTest.java     |  2 +-
 .../runners/dataflow/DataflowRunnerTest.java  |  2 +-
 .../dataflow/TestDataflowRunnerTest.java      |  2 +-
 .../options/DataflowPipelineOptionsTest.java  |  2 +-
 .../transforms/DataflowGroupByKeyTest.java    |  2 +-
 .../dataflow/transforms/DataflowViewTest.java |  2 +-
 .../beam/sdk/util/NoopPathValidator.java      | 51 -------------------
 .../gcp/options/GcpOptionsTest.java           |  2 +-
 9 files changed, 8 insertions(+), 59 deletions(-)
 delete mode 100644 sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java

diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
index 7e883005aff7..bca3428132e9 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
@@ -39,9 +39,9 @@
 import java.math.BigDecimal;
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
index 3c261d041e07..a670fac81b8a 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
@@ -53,13 +53,13 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.testing.ExpectedLogs;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.BackOffAdapter;
 import org.apache.beam.sdk.util.FastNanoClockAndSleeper;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index 6848b85cfa33..ce01aa11ba2b 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -70,6 +70,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.extensions.gcp.auth.NoopCredentialFactory;
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -83,7 +84,6 @@
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.GcsUtil;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.util.ReleaseInfo;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
index bf157474b677..883d3443d75f 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
@@ -51,13 +51,13 @@
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.SerializableMatcher;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.util.Transport;
 import org.apache.beam.sdk.values.PCollection;
 import org.hamcrest.BaseMatcher;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
index 8b8fd6c83c96..613604ac8426 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
@@ -22,11 +22,11 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage;
 
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.ResetDateTimeProvider;
 import org.apache.beam.sdk.testing.RestoreSystemProperties;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
index fc2ff3d284e7..737b40813f33 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java
@@ -26,13 +26,13 @@
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.Sessions;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
index f0dbd6183e60..dea96b96ea21 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java
@@ -21,6 +21,7 @@
 import org.apache.beam.runners.dataflow.DataflowRunner;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -28,7 +29,6 @@
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
deleted file mode 100644
index 85b68b21cfdf..000000000000
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/NoopPathValidator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
-import org.apache.beam.sdk.io.fs.ResourceId;
-import org.apache.beam.sdk.options.PipelineOptions;
-
-/**
- * @deprecated use {@link org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator}.
- */
-@Deprecated
-public class NoopPathValidator implements PathValidator {
-
-  private NoopPathValidator() {
-  }
-
-  public static PathValidator fromOptions(
-      @SuppressWarnings("unused") PipelineOptions options) {
-    return new NoopPathValidator();
-  }
-
-  @Override
-  public void validateInputFilePatternSupported(String filepattern) {}
-
-  @Override
-  public void validateOutputFilePrefixSupported(String filePrefix) {}
-
-  @Override
-  public void validateOutputResourceSupported(ResourceId resourceId) {}
-
-  @Override
-  public String verifyPath(String path) {
-    return path;
-  }
-}
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
index 3cf698030a0d..fdf3e1aa2fbb 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
@@ -41,11 +41,11 @@
 import org.apache.beam.sdk.extensions.gcp.auth.TestCredential;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions.DefaultProjectFactory;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions.GcpTempLocationFactory;
+import org.apache.beam.sdk.extensions.gcp.storage.NoopPathValidator;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.RestoreSystemProperties;
 import org.apache.beam.sdk.util.GcsUtil;
-import org.apache.beam.sdk.util.NoopPathValidator;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.junit.Before;
 import org.junit.Rule;

From 0b1595640686b5e9e6314a4868b923535d253c37 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 18:04:01 -0700
Subject: [PATCH 169/387] Improve DirectRunner Javadoc

---
 .../beam/runners/direct/DirectOptions.java    |  2 +-
 .../beam/runners/direct/DirectRegistrar.java  | 10 +++---
 .../beam/runners/direct/DirectRunner.java     | 33 ++++++++++++-------
 3 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
index 3b66cc661aae..574ab46fb449 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
@@ -49,7 +49,7 @@ public interface DirectOptions extends PipelineOptions, ApplicationNameOptions {
   @Default.Boolean(true)
   @Description(
       "Controls whether the DirectRunner should ensure that all of the elements of every "
-          + "PCollection are encodable. All elements in a PCollection must be encodable.")
+          + "PCollection can be encoded and decoded by that PCollection's Coder.")
   boolean isEnforceEncodability();
   void setEnforceEncodability(boolean test);
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
index 3e69e2b527d2..0e6fbab88820 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
@@ -26,31 +26,31 @@
 
 /**
  * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the
- * {@link org.apache.beam.runners.direct.DirectRunner}.
+ * {@link DirectRunner}.
  */
 public class DirectRegistrar {
   private DirectRegistrar() {}
   /**
-   * Registers the {@link org.apache.beam.runners.direct.DirectRunner}.
+   * Registers the {@link DirectRunner}.
    */
   @AutoService(PipelineRunnerRegistrar.class)
   public static class Runner implements PipelineRunnerRegistrar {
     @Override
     public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
       return ImmutableList.<Class<? extends PipelineRunner<?>>>of(
-          org.apache.beam.runners.direct.DirectRunner.class);
+          DirectRunner.class);
     }
   }
 
   /**
-   * Registers the {@link org.apache.beam.runners.direct.DirectOptions}.
+   * Registers the {@link DirectOptions}.
    */
   @AutoService(PipelineOptionsRegistrar.class)
   public static class Options implements PipelineOptionsRegistrar {
     @Override
     public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
       return ImmutableList.<Class<? extends PipelineOptions>>of(
-          org.apache.beam.runners.direct.DirectOptions.class);
+          DirectOptions.class);
     }
   }
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index b0ce5eb02845..181896f3b3fc 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -51,8 +51,14 @@
 import org.joda.time.Duration;
 
 /**
- * An In-Memory implementation of the Dataflow Programming Model. Supports Unbounded
- * {@link PCollection PCollections}.
+ * A {@link PipelineRunner} that executes a {@link Pipeline} within the process that constructed the
+ * {@link Pipeline}.
+ *
+ * <p>The {@link DirectRunner} is suitable for running a {@link Pipeline} on small scale, example,
+ * and test data, and should be used for ensuring that processing logic is correct. It also
+ * is appropriate for executing unit tests and performs additional work to ensure that behavior
+ * contained within a {@link Pipeline} does not break assumptions within the Beam model, to improve
+ * the ability to execute a {@link Pipeline} at scale on a distributed backend.
  */
 public class DirectRunner extends PipelineRunner<DirectPipelineResult> {
 
@@ -127,6 +133,9 @@ static BundleFactory bundleFactoryFor(
   private final Set<Enforcement> enabledEnforcements;
   private Supplier<Clock> clockSupplier = new NanosOffsetClockSupplier();
 
+  /**
+   * Construct a {@link DirectRunner} from the provided options.
+   */
   public static DirectRunner fromOptions(PipelineOptions options) {
     return new DirectRunner(options.as(DirectOptions.class));
   }
@@ -246,8 +255,6 @@ private List<PTransformOverride> defaultTransformOverrides() {
 
   /**
    * The result of running a {@link Pipeline} with the {@link DirectRunner}.
-   *
-   * <p>Throws {@link UnsupportedOperationException} for all methods.
    */
   public static class DirectPipelineResult implements PipelineResult {
     private final PipelineExecutor executor;
@@ -274,14 +281,11 @@ public MetricResults metrics() {
     }
 
     /**
-     * Blocks until the {@link Pipeline} execution represented by this
-     * {@link DirectPipelineResult} is complete, returning the terminal state.
+     * {@inheritDoc}.
      *
-     * <p>If the pipeline terminates abnormally by throwing an exception, this will rethrow the
-     * exception. Future calls to {@link #getState()} will return
-     * {@link org.apache.beam.sdk.PipelineResult.State#FAILED}.
-     *
-     * <p>See also {@link PipelineExecutor#waitUntilFinish(Duration)}.
+     * <p>If the pipeline terminates abnormally by throwing an {@link Exception}, this will rethrow
+     * the original {@link Exception}. Future calls to {@link #getState()} will return {@link
+     * org.apache.beam.sdk.PipelineResult.State#FAILED}.
      */
     @Override
     public State waitUntilFinish() {
@@ -298,6 +302,13 @@ public State cancel() {
       return executor.getPipelineState();
     }
 
+    /**
+     * {@inheritDoc}.
+     *
+     * <p>If the pipeline terminates abnormally by throwing an {@link Exception}, this will rethrow
+     * the original {@link Exception}. Future calls to {@link #getState()} will return {@link
+     * org.apache.beam.sdk.PipelineResult.State#FAILED}.
+     */
     @Override
     public State waitUntilFinish(Duration duration) {
       State startState = this.state;

From 01b3f87f977d44eac23eb5488074bbc638858a9d Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 18:09:12 -0700
Subject: [PATCH 170/387] Update Coder Documentation

---
 .../apache/beam/sdk/coders/AtomicCoder.java   | 17 ++++++++++++--
 .../org/apache/beam/sdk/coders/Coder.java     | 22 +++++++------------
 .../apache/beam/sdk/coders/CustomCoder.java   | 17 +++++---------
 .../beam/sdk/coders/StructuredCoder.java      | 10 +++++----
 4 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
index 7bcd5327c014..b244ed5f0d1e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AtomicCoder.java
@@ -35,14 +35,22 @@ public abstract class AtomicCoder<T> extends StructuredCoder<T> {
   /**
    * {@inheritDoc}.
    *
-   * @throws NonDeterministicException
+   * <p>Unless overridden, does not throw. An {@link AtomicCoder} is presumed to be deterministic
+   *
+   * @throws NonDeterministicException if overridden to indicate that this sublcass of
+   *         {@link AtomicCoder} is not deterministic
    */
   @Override
   public void verifyDeterministic() throws NonDeterministicException {}
 
+  /**
+   * {@inheritDoc}.
+   *
+   * @return the empty list
+   */
   @Override
   public List<? extends Coder<?>> getCoderArguments() {
-    return null;
+    return Collections.emptyList();
   }
 
   /**
@@ -65,6 +73,11 @@ public final boolean equals(Object other) {
     return other != null && this.getClass().equals(other.getClass());
   }
 
+  /**
+   * {@inheritDoc}.
+   *
+   * @return the {@link #hashCode()} of the {@link Class} of this {@link AtomicCoder}.
+   */
   @Override
   public final int hashCode() {
     return this.getClass().hashCode();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
index 2ee532dfaa6e..edcc3a81c31d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java
@@ -43,23 +43,17 @@
  * byte streams.
  *
  * <p>{@link Coder} instances are serialized during job creation and deserialized
- * before use, via JSON serialization. See {@link SerializableCoder} for an example of a
- * {@link Coder} that adds a custom field to
- * the {@link Coder} serialization. It provides a constructor annotated with
- * {@link com.fasterxml.jackson.annotation.JsonCreator}, which is a factory method used when
- * deserializing a {@link Coder} instance.
+ * before use. This will generally be performed by serializing the object via Java Serialization.
  *
  * <p>{@link Coder} classes for compound types are often composed from coder classes for types
  * contains therein. The composition of {@link Coder} instances into a coder for the compound
  * class is the subject of the {@link CoderProvider} type, which enables automatic generic
- * composition of {@link Coder} classes within the {@link CoderRegistry}. With particular
- * static methods on a compound {@link Coder} class, a {@link CoderProvider} can be automatically
- * inferred. See {@link KvCoder} for an example of a simple compound {@link Coder} that supports
- * automatic composition in the {@link CoderRegistry}.
+ * composition of {@link Coder} classes within the {@link CoderRegistry}. See {@link CoderProvider}
+ * and {@link CoderRegistry} for more information about how coders are inferred.
  *
  * <p>All methods of a {@link Coder} are required to be thread safe.
  *
- * @param <T> the type of the values being transcoded
+ * @param <T> the type of values being encoded and decoded
  */
 public abstract class Coder<T> implements Serializable {
   /** The context in which encoding or decoding is being done. */
@@ -167,10 +161,10 @@ public T decode(InputStream inStream, Context context)
   }
 
   /**
-   * If this is a {@code Coder} for a parameterized type, returns the
-   * list of {@code Coder}s being used for each of the parameters, or
-   * returns {@code null} if this cannot be done or this is not a
-   * parameterized type.
+   * If this is a {@link Coder} for a parameterized type, returns the
+   * list of {@link Coder}s being used for each of the parameters in the same order they appear
+   * within the parameterized type's type signature. If this cannot be done, or this
+   * {@link Coder} does not encode/decode a parameterized type, returns the empty list.
    */
   public abstract List<? extends Coder<?>> getCoderArguments();
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
index c58192373432..e8ce5b1d538f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CustomCoder.java
@@ -22,19 +22,12 @@
 import java.util.List;
 
 /**
- * An abstract base class for writing a {@link Coder} class that encodes itself via Java
- * serialization.
+ * An abstract base class that implements all methods of {@link Coder} except {@link Coder#encode}
+ * and {@link Coder#decode}.
  *
- * <p>To complete an implementation, subclasses must implement {@link Coder#encode}
- * and {@link Coder#decode} methods.
- *
- * <p>Not to be confused with {@link SerializableCoder} that encodes objects that implement the
- * {@link Serializable} interface.
- *
- * @param <T> the type of elements handled by this coder
+ * @param <T> the type of values being encoded and decoded
  */
-public abstract class CustomCoder<T> extends Coder<T>
-    implements Serializable {
+public abstract class CustomCoder<T> extends Coder<T> implements Serializable {
 
   /**
    * {@inheritDoc}.
@@ -61,5 +54,5 @@ public void verifyDeterministic() throws NonDeterministicException {
 
   // This coder inherits isRegisterByteSizeObserverCheap,
   // getEncodedElementByteSize and registerByteSizeObserver
-  // from StructuredCoder. Override if we can do better.
+  // from Coder. Override if we can do better.
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
index 42c0598aa985..a145215608d0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
@@ -33,11 +33,11 @@
  * <p>To extend {@link StructuredCoder}, override the following methods as appropriate:
  *
  * <ul>
- *   <li>{@link #getComponents}: the default implementation returns {@link #getCoderArguments}.</li>
- *   <li>{@link #getEncodedElementByteSize} and
- *       {@link #isRegisterByteSizeObserverCheap}: the
+ *   <li>{@link #getComponents}: the default implementation returns {@link #getCoderArguments}.
+ *   <li>{@link #getEncodedElementByteSize} and {@link #isRegisterByteSizeObserverCheap}: the
  *       default implementation encodes values to bytes and counts the bytes, which is considered
- *       expensive.</li>
+ *       expensive. The default element byte size observer uses the value returned by
+ *       {@link #getEncodedElementByteSize}.
  * </ul>
  */
 public abstract class StructuredCoder<T> extends Coder<T> {
@@ -45,6 +45,8 @@ protected StructuredCoder() {}
 
   /**
    * Returns the list of {@link Coder Coders} that are components of this {@link Coder}.
+   *
+   * <p>The default components will be equal to the value returned by {@link #getCoderArguments()}.
    */
   public List<? extends Coder<?>> getComponents() {
     List<? extends Coder<?>> coderArguments = getCoderArguments();

From 439fd202f59ce84ef3f1cf5bd6d076deb949e93d Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 18:09:37 -0700
Subject: [PATCH 171/387] Remove verifyDeterministic from StructuredCoder

This is replaced within all uses in the Core SDK by
Coder.verifyDeterministic(Coder, String, List<Coder>), which is a
general utility.
---
 .../apache/beam/sdk/coders/StructuredCoder.java | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
index a145215608d0..2eb662ba280c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuredCoder.java
@@ -18,7 +18,6 @@
 package org.apache.beam.sdk.coders;
 
 import java.io.ByteArrayOutputStream;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import org.apache.beam.sdk.values.TypeDescriptor;
@@ -100,22 +99,6 @@ public String toString() {
     return builder.toString();
   }
 
-  protected void verifyDeterministic(String message, Iterable<Coder<?>> coders)
-      throws NonDeterministicException {
-    for (Coder<?> coder : coders) {
-      try {
-        coder.verifyDeterministic();
-      } catch (NonDeterministicException e) {
-        throw new NonDeterministicException(this, message, e);
-      }
-    }
-  }
-
-  protected void verifyDeterministic(String message, Coder<?>... coders)
-      throws NonDeterministicException {
-    verifyDeterministic(message, Arrays.asList(coders));
-  }
-
   /**
    * {@inheritDoc}
    *

From 758fee8e95bbbda985988b6ea92f6c7a741bf74d Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Tue, 9 May 2017 16:45:24 -0700
Subject: [PATCH 172/387] Use text output for first two mobile gaming examples

---
 .../complete/game/HourlyTeamScore.java        |  57 ++----
 .../examples/complete/game/LeaderBoard.java   |  26 +++
 .../examples/complete/game/UserScore.java     |  56 ++----
 .../complete/game/utils/WriteToText.java      | 184 ++++++++++++++++++
 4 files changed, 251 insertions(+), 72 deletions(-)
 create mode 100644 examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java

diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
index 2928882a1776..6a322da8fc87 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
@@ -20,9 +20,8 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.TimeZone;
-import org.apache.beam.examples.complete.game.utils.WriteWindowedToBigQuery;
+import org.apache.beam.examples.complete.game.utils.WriteToText;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
@@ -58,12 +57,11 @@
  * like this:
  * <pre>{@code
  *   --project=YOUR_PROJECT_ID
- *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowRunner
- *   --dataset=YOUR-DATASET
+ *   --tempLocation=YOUR_TEMP_DIRECTORY
+ *   --runner=YOUR_RUNNER
+ *   --output=YOUR_OUTPUT_DIRECTORY
  * }
  * </pre>
- * where the BigQuery dataset you specify must already exist.
  *
  * <p>Optionally include {@code --input} to specify the batch input file path.
  * To indicate a time after which the data should be filtered out, include the
@@ -107,39 +105,26 @@ interface Options extends UserScore.Options {
     @Default.String("2100-01-01-00-00")
     String getStopMin();
     void setStopMin(String value);
-
-    @Description("The BigQuery table name. Should not already exist.")
-    @Default.String("hourly_team_score")
-    String getHourlyTeamScoreTableName();
-    void setHourlyTeamScoreTableName(String value);
   }
 
   /**
-   * Create a map of information that describes how to write pipeline output to BigQuery. This map
-   * is passed to the {@link WriteWindowedToBigQuery} constructor to write team score sums and
+   * Create a map of information that describes how to write pipeline output to text. This map
+   * is passed to the {@link WriteToText} constructor to write team score sums and
    * includes information about window start time.
    */
-  protected static Map<String, WriteWindowedToBigQuery.FieldInfo<KV<String, Integer>>>
-      configureWindowedTableWrite() {
-    Map<String, WriteWindowedToBigQuery.FieldInfo<KV<String, Integer>>> tableConfig =
-        new HashMap<String, WriteWindowedToBigQuery.FieldInfo<KV<String, Integer>>>();
-    tableConfig.put(
-        "team",
-        new WriteWindowedToBigQuery.FieldInfo<KV<String, Integer>>(
-            "STRING", (c, w) -> c.element().getKey()));
-    tableConfig.put(
-        "total_score",
-        new WriteWindowedToBigQuery.FieldInfo<KV<String, Integer>>(
-            "INTEGER", (c, w) -> c.element().getValue()));
-    tableConfig.put(
+  protected static Map<String, WriteToText.FieldFn<KV<String, Integer>>>
+      configureOutput() {
+    Map<String, WriteToText.FieldFn<KV<String, Integer>>> config =
+        new HashMap<String, WriteToText.FieldFn<KV<String, Integer>>>();
+    config.put("team", (c, w) -> c.element().getKey());
+    config.put("total_score", (c, w) -> c.element().getValue());
+    config.put(
         "window_start",
-        new WriteWindowedToBigQuery.FieldInfo<KV<String, Integer>>(
-            "STRING",
-            (c, w) -> {
+        (c, w) -> {
               IntervalWindow window = (IntervalWindow) w;
               return fmt.print(window.start());
-            }));
-    return tableConfig;
+            });
+    return config;
   }
 
 
@@ -186,12 +171,10 @@ public static void main(String[] args) throws Exception {
       // Extract and sum teamname/score pairs from the event data.
       .apply("ExtractTeamScore", new ExtractAndSumScore("team"))
       .apply("WriteTeamScoreSums",
-        new WriteWindowedToBigQuery<KV<String, Integer>>(
-            options.as(GcpOptions.class).getProject(),
-            options.getDataset(),
-            options.getHourlyTeamScoreTableName(),
-            configureWindowedTableWrite()));
-
+          new WriteToText<KV<String, Integer>>(
+              options.getOutput(),
+              configureOutput(),
+              true));
 
     pipeline.run().waitUntilFinish();
   }
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
index bfad9f65fc3f..f673a8de688a 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
@@ -106,6 +106,11 @@ public class LeaderBoard extends HourlyTeamScore {
    */
   interface Options extends HourlyTeamScore.Options, ExampleOptions, StreamingOptions {
 
+    @Description("BigQuery Dataset to write tables to. Must already exist.")
+    @Validation.Required
+    String getDataset();
+    void setDataset(String value);
+
     @Description("Pub/Sub topic to read from")
     @Validation.Required
     String getTopic();
@@ -163,6 +168,27 @@ interface Options extends HourlyTeamScore.Options, ExampleOptions, StreamingOpti
     return tableConfigure;
   }
 
+
+  /**
+   * Create a map of information that describes how to write pipeline output to BigQuery. This map
+   * is passed to the {@link WriteToBigQuery} constructor to write user score sums.
+   */
+  protected static Map<String, WriteToBigQuery.FieldInfo<KV<String, Integer>>>
+  configureBigQueryWrite() {
+    Map<String, WriteToBigQuery.FieldInfo<KV<String, Integer>>> tableConfigure =
+        new HashMap<String, WriteToBigQuery.FieldInfo<KV<String, Integer>>>();
+    tableConfigure.put(
+        "user",
+        new WriteToBigQuery.FieldInfo<KV<String, Integer>>(
+            "STRING", (c, w) -> c.element().getKey()));
+    tableConfigure.put(
+        "total_score",
+        new WriteToBigQuery.FieldInfo<KV<String, Integer>>(
+            "INTEGER", (c, w) -> c.element().getValue()));
+    return tableConfigure;
+  }
+
+
   /**
    * Create a map of information that describes how to write pipeline output to BigQuery. This map
    * is used to write user score sums.
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index 8110146d7020..7297bcdf3e94 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -20,11 +20,10 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.avro.reflect.Nullable;
-import org.apache.beam.examples.complete.game.utils.WriteToBigQuery;
+import org.apache.beam.examples.complete.game.utils.WriteToText;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.DefaultCoder;
-import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
@@ -62,9 +61,9 @@
  * the pipeline configuration like this:
  * <pre>{@code
  *   --project=YOUR_PROJECT_ID
- *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowRunner
- *   --dataset=YOUR-DATASET
+ *   --tempLocation=YOUR_TEMP_DIRECTORY
+ *   --runner=YOUR_RUNNER
+ *   --output=YOUR_OUTPUT_DIRECTORY
  * }
  * </pre>
  * where the BigQuery dataset you specify must already exist.
@@ -186,37 +185,26 @@ public interface Options extends PipelineOptions {
     String getInput();
     void setInput(String value);
 
-    @Description("BigQuery Dataset to write tables to. Must already exist.")
+    // Set this required option to specify where to write the output.
+    @Description("Path of the file to write to.")
     @Validation.Required
-    String getDataset();
-    void setDataset(String value);
-
-    @Description("The BigQuery table name. Should not already exist.")
-    @Default.String("user_score")
-    String getUserScoreTableName();
-    void setUserScoreTableName(String value);
+    String getOutput();
+    void setOutput(String value);
   }
 
   /**
-   * Create a map of information that describes how to write pipeline output to BigQuery. This map
-   * is passed to the {@link WriteToBigQuery} constructor to write user score sums.
+   * Create a map of information that describes how to write pipeline output to text. This map
+   * is passed to the {@link WriteToText} constructor to write user score sums.
    */
-  protected static Map<String, WriteToBigQuery.FieldInfo<KV<String, Integer>>>
-      configureBigQueryWrite() {
-    Map<String, WriteToBigQuery.FieldInfo<KV<String, Integer>>> tableConfigure =
-        new HashMap<String, WriteToBigQuery.FieldInfo<KV<String, Integer>>>();
-    tableConfigure.put(
-        "user",
-        new WriteToBigQuery.FieldInfo<KV<String, Integer>>(
-            "STRING", (c, w) -> c.element().getKey()));
-    tableConfigure.put(
-        "total_score",
-        new WriteToBigQuery.FieldInfo<KV<String, Integer>>(
-            "INTEGER", (c, w) -> c.element().getValue()));
-    return tableConfigure;
+  protected static Map<String, WriteToText.FieldFn<KV<String, Integer>>>
+      configureOutput() {
+    Map<String, WriteToText.FieldFn<KV<String, Integer>>> config =
+        new HashMap<String, WriteToText.FieldFn<KV<String, Integer>>>();
+    config.put("user", (c, w) -> c.element().getKey());
+    config.put("total_score", (c, w) -> c.element().getValue());
+    return config;
   }
 
-
   /**
    * Run a batch pipeline.
    */
@@ -234,15 +222,13 @@ public static void main(String[] args) throws Exception {
         .apply("ExtractUserScore", new ExtractAndSumScore("user"))
         .apply(
             "WriteUserScoreSums",
-            new WriteToBigQuery<KV<String, Integer>>(
-                options.as(GcpOptions.class).getProject(),
-                options.getDataset(),
-                options.getUserScoreTableName(),
-                configureBigQueryWrite()));
+            new WriteToText<KV<String, Integer>>(
+                options.getOutput(),
+                configureOutput(),
+                false));
 
     // Run the batch pipeline.
     pipeline.run().waitUntilFinish();
   }
   // [END DocInclude_USMain]
-
 }
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java
new file mode 100644
index 000000000000..e6c8ddbecc41
--- /dev/null
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.examples.complete.game.utils;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Verify.verifyNotNull;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.stream.Collectors;
+import org.apache.beam.sdk.io.FileBasedSink;
+import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
+import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/**
+ * Generate, format, and write rows. Use provided information about the field names and types, as
+ * well as lambda functions that describe how to generate their values.
+ */
+public class WriteToText<InputT>
+    extends PTransform<PCollection<InputT>, PDone> {
+
+  private static final DateTimeFormatter formatter =
+      DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS")
+          .withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("PST")));
+
+  protected String filenamePrefix;
+  protected Map<String, FieldFn<InputT>> fieldFn;
+  protected boolean windowed;
+
+  public WriteToText() {
+  }
+
+  public WriteToText(
+      String filenamePrefix,
+      Map<String, FieldFn<InputT>> fieldFn,
+      boolean windowed) {
+    this.filenamePrefix = filenamePrefix;
+    this.fieldFn = fieldFn;
+    this.windowed = windowed;
+  }
+
+  /**
+   * A {@link Serializable} function from a {@link DoFn.ProcessContext}
+   * and {@link BoundedWindow} to the value for that field.
+   */
+  public interface FieldFn<InputT> extends Serializable {
+    Object apply(DoFn<InputT, String>.ProcessContext context, BoundedWindow window);
+  }
+
+  /** Convert each key/score pair into a row as specified by fieldFn. */
+  protected class BuildRowFn extends DoFn<InputT, String> {
+
+    @ProcessElement
+    public void processElement(ProcessContext c, BoundedWindow window) {
+      List<String> fields = new ArrayList<String>();
+      for (Map.Entry<String, FieldFn<InputT>> entry : fieldFn.entrySet()) {
+        String key = entry.getKey();
+        FieldFn<InputT> fcn = entry.getValue();
+        fields.add(key + ": " + fcn.apply(c, window));
+      }
+      String result = fields.stream().collect(Collectors.joining(", "));
+      c.output(result);
+    }
+  }
+
+  /**
+   * A {@link DoFn} that writes elements to files with names deterministically derived from the
+   * lower and upper bounds of their key (an {@link IntervalWindow}).
+   */
+  protected class WriteOneFilePerWindow extends PTransform<PCollection<String>, PDone> {
+
+    private final String filenamePrefix;
+
+    public WriteOneFilePerWindow(String filenamePrefix) {
+      this.filenamePrefix = filenamePrefix;
+    }
+
+    @Override
+    public PDone expand(PCollection<String> input) {
+      // Verify that the input has a compatible window type.
+      checkArgument(
+          input.getWindowingStrategy().getWindowFn().windowCoder() == IntervalWindow.getCoder());
+
+      // filenamePrefix may contain a directory and a filename component. Pull out only the filename
+      // component from that path for the PerWindowFiles.
+      String prefix = "";
+      ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix);
+      if (!resource.isDirectory()) {
+        prefix = verifyNotNull(
+            resource.getFilename(),
+            "A non-directory resource should have a non-null filename: %s",
+            resource);
+      }
+
+      return input.apply(
+          TextIO.write()
+              .to(resource.getCurrentDirectory())
+              .withFilenamePolicy(new PerWindowFiles(prefix))
+              .withWindowedWrites()
+              .withNumShards(3));
+    }
+  }
+
+  /**
+   * A {@link FilenamePolicy} produces a base file name for a write based on metadata about the data
+   * being written. This always includes the shard number and the total number of shards. For
+   * windowed writes, it also includes the window and pane index (a sequence number assigned to each
+   * trigger firing).
+   */
+  protected static class PerWindowFiles extends FilenamePolicy {
+
+    private final String prefix;
+
+    public PerWindowFiles(String prefix) {
+      this.prefix = prefix;
+    }
+
+    public String filenamePrefixForWindow(IntervalWindow window) {
+      return String.format("%s-%s-%s",
+          prefix, formatter.print(window.start()), formatter.print(window.end()));
+    }
+
+    @Override
+    public ResourceId windowedFilename(
+        ResourceId outputDirectory, WindowedContext context, String extension) {
+      IntervalWindow window = (IntervalWindow) context.getWindow();
+      String filename = String.format(
+          "%s-%s-of-%s%s",
+          filenamePrefixForWindow(window), context.getShardNumber(), context.getNumShards(),
+          extension);
+      return outputDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+    }
+
+    @Override
+    public ResourceId unwindowedFilename(
+        ResourceId outputDirectory, Context context, String extension) {
+      throw new UnsupportedOperationException("Unsupported.");
+    }
+  }
+
+  @Override
+  public PDone expand(PCollection<InputT> teamAndScore) {
+    if (windowed) {
+      teamAndScore
+          .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
+          .apply(new WriteToText.WriteOneFilePerWindow(filenamePrefix));
+    } else {
+      teamAndScore
+          .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
+          .apply(TextIO.write().to(filenamePrefix));
+    }
+    return PDone.in(teamAndScore.getPipeline());
+  }
+}

From 4a5133aa31e37598139b81650ab6025d80f14800 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Wed, 10 May 2017 10:11:53 -0700
Subject: [PATCH 173/387] [BEAM-2249] Correctly handle partial reads in
 AvroSource

---
 .../org/apache/beam/sdk/io/AvroSource.java    | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
index 61bc4a4f1563..37bbe46e3a3b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
@@ -59,6 +59,7 @@
 import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
 import org.apache.commons.compress.utils.CountingInputStream;
+import org.apache.commons.compress.utils.IOUtils;
 
 // CHECKSTYLE.OFF: JavadocStyle
 /**
@@ -124,7 +125,7 @@ public class AvroSource<T> extends BlockBasedSource<T> {
   // Default minimum bundle size (chosen as two default-size Avro blocks to attempt to
   // ensure that every source has at least one block of records).
   // The default sync interval is 64k.
-  static final long DEFAULT_MIN_BUNDLE_SIZE = 2 * DataFileConstants.DEFAULT_SYNC_INTERVAL;
+  private static final long DEFAULT_MIN_BUNDLE_SIZE = 2 * DataFileConstants.DEFAULT_SYNC_INTERVAL;
 
   // The JSON schema used to encode records.
   private final String readSchemaString;
@@ -663,23 +664,27 @@ public boolean readNextBlock() throws IOException {
       long headerSize = countStream.getBytesRead() - preHeaderCount;
 
       // Create the current block by reading blockSize bytes. Block sizes permitted by the Avro
-      // specification are [32, 2^30], so this narrowing is ok.
+      // specification are [32, 2^30], so the cast is safe.
       byte[] data = new byte[(int) blockSize];
-      int read = stream.read(data);
-      checkState(blockSize == read, "Only %s/%s bytes in the block were read", read, blockSize);
+      int bytesRead = IOUtils.readFully(stream, data);
+      checkState(
+          blockSize == bytesRead,
+          "Only able to read %s/%s bytes in the block before EOF reached.",
+          bytesRead,
+          blockSize);
       currentBlock = new AvroBlock<>(data, numRecords, getCurrentSource());
 
       // Read the end of this block, which MUST be a sync marker for correctness.
       byte[] syncMarker = getCurrentSource().getSyncMarker();
       byte[] readSyncMarker = new byte[syncMarker.length];
       long syncMarkerOffset = startOfNextBlock + headerSize + blockSize;
-      long bytesRead = stream.read(readSyncMarker);
+      bytesRead = IOUtils.readFully(stream, readSyncMarker);
       checkState(
           bytesRead == syncMarker.length,
-          "When trying to read a sync marker at position %s, only able to read %s/%s bytes",
-          syncMarkerOffset,
+          "Only able to read %s/%s bytes of Avro sync marker at position %s before EOF reached.",
           bytesRead,
-          syncMarker.length);
+          syncMarker.length,
+          syncMarkerOffset);
       if (!Arrays.equals(syncMarker, readSyncMarker)) {
         throw new IllegalStateException(
             String.format(

From ab6824d35ce3be481f98efed4b65a8dd264652d6 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 10 May 2017 12:04:20 -0700
Subject: [PATCH 174/387] [BEAM-2250] remove experimental and internal things
 from pydoc

---
 sdks/python/generate_pydoc.sh | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/sdks/python/generate_pydoc.sh b/sdks/python/generate_pydoc.sh
index 6039942e7eda..1fea6f12a1c5 100755
--- a/sdks/python/generate_pydoc.sh
+++ b/sdks/python/generate_pydoc.sh
@@ -31,21 +31,24 @@ rm -rf target/docs/*
 
 mkdir -p target/docs/source
 
-# Exclude autogenerated API message definition files that aren't part of SDK.
-excluded_internal_clients=(
+# Exclude internal/experimental files from the documentation.
+excluded_internal_code=(
+    apache_beam/examples/
     apache_beam/internal/clients/
     apache_beam/io/gcp/internal/clients/
+    apache_beam/runners/api/
+    apache_beam/runners/test/
+    apache_beam/runners/portability/
+    apache_beam/runners/worker/
     apache_beam/runners/dataflow/internal/clients/
-    apache_beam/examples/complete/juliaset/setup.py)
+    apache_beam/testing/data/)
 
 python $(type -p sphinx-apidoc) -f -o target/docs/source apache_beam \
-    "${excluded_internal_clients[@]}" "*_test.py"
+    "${excluded_internal_code[@]}" "*_test.py"
 
 # Remove Cython modules from doc template; they won't load
 sed -i -e '/.. automodule:: apache_beam.coders.stream/d' \
     target/docs/source/apache_beam.coders.rst
-sed -i -e '/.. automodule:: apache_beam.runners.worker.statesampler/d' \
-    target/docs/source/apache_beam.runners.worker.rst
 
 # Create the configuration and index files
 cat > target/docs/source/conf.py <<'EOF'

From 5bbc04232de6c6b9dce2a82b1449ffa03c83da47 Mon Sep 17 00:00:00 2001
From: Reuven Lax <relax@google.com>
Date: Tue, 9 May 2017 13:02:12 -0700
Subject: [PATCH 175/387] Renames FileBasedSink inner classes

FileBasedWriteOperation -> WriteOperation, FileBasedWriter -> Writer
---
 .../construction/PTransformMatchersTest.java  |  2 +-
 .../direct/WriteWithShardingFactoryTest.java  |  2 +-
 .../java/org/apache/beam/sdk/io/AvroSink.java | 14 ++---
 .../org/apache/beam/sdk/io/FileBasedSink.java | 62 +++++++++----------
 .../org/apache/beam/sdk/io/TFRecordIO.java    | 16 ++---
 .../java/org/apache/beam/sdk/io/TextSink.java | 14 ++---
 .../org/apache/beam/sdk/io/WriteFiles.java    | 44 ++++++-------
 .../apache/beam/sdk/io/FileBasedSinkTest.java | 20 +++---
 .../org/apache/beam/sdk/io/SimpleSink.java    |  4 +-
 .../org/apache/beam/sdk/io/xml/XmlSink.java   |  8 +--
 10 files changed, 93 insertions(+), 93 deletions(-)

diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index 9ae236be0737..cfea62ffb537 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -510,7 +510,7 @@ public void writeWithRunnerDeterminedSharding() {
         WriteFiles.to(
             new FileBasedSink<Integer>(StaticValueProvider.of(outputDirectory), policy) {
               @Override
-              public FileBasedWriteOperation<Integer> createWriteOperation() {
+              public WriteOperation<Integer> createWriteOperation() {
                 return null;
               }
             });
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
index 5a2a328beab0..5c4fea1cb753 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
@@ -134,7 +134,7 @@ public void withNoShardingSpecifiedReturnsNewTransform() {
     WriteFiles<Object> original = WriteFiles.to(
         new FileBasedSink<Object>(StaticValueProvider.of(outputDirectory), policy) {
           @Override
-          public FileBasedWriteOperation<Object> createWriteOperation() {
+          public WriteOperation<Object> createWriteOperation() {
             throw new IllegalArgumentException("Should not be used");
           }
         });
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java
index 7d42574447b9..6c362664d8fa 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java
@@ -51,12 +51,12 @@ class AvroSink<T> extends FileBasedSink<T> {
   }
 
   @Override
-  public FileBasedWriteOperation<T> createWriteOperation() {
+  public WriteOperation<T> createWriteOperation() {
     return new AvroWriteOperation<>(this, coder, codec, metadata);
   }
 
-  /** A {@link FileBasedWriteOperation FileBasedWriteOperation} for Avro files. */
-  private static class AvroWriteOperation<T> extends FileBasedWriteOperation<T> {
+  /** A {@link WriteOperation WriteOperation} for Avro files. */
+  private static class AvroWriteOperation<T> extends WriteOperation<T> {
     private final AvroCoder<T> coder;
     private final SerializableAvroCodecFactory codec;
     private final ImmutableMap<String, Object> metadata;
@@ -72,19 +72,19 @@ private AvroWriteOperation(AvroSink<T> sink,
     }
 
     @Override
-    public FileBasedWriter<T> createWriter() throws Exception {
+    public Writer<T> createWriter() throws Exception {
       return new AvroWriter<>(this, coder, codec, metadata);
     }
   }
 
-  /** A {@link FileBasedWriter FileBasedWriter} for Avro files. */
-  private static class AvroWriter<T> extends FileBasedWriter<T> {
+  /** A {@link Writer Writer} for Avro files. */
+  private static class AvroWriter<T> extends Writer<T> {
     private final AvroCoder<T> coder;
     private DataFileWriter<T> dataFileWriter;
     private SerializableAvroCodecFactory codec;
     private final ImmutableMap<String, Object> metadata;
 
-    public AvroWriter(FileBasedWriteOperation<T> writeOperation,
+    public AvroWriter(WriteOperation<T> writeOperation,
                       AvroCoder<T> coder,
                       SerializableAvroCodecFactory codec,
                       ImmutableMap<String, Object> metadata) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 32aa9c388d89..c2e230d47e60 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -78,7 +78,7 @@
  * type, etc.).
  *
  * <p>At pipeline construction time, the methods of FileBasedSink are called to validate the sink
- * and to create a {@link FileBasedWriteOperation} that manages the process of writing to the sink.
+ * and to create a {@link WriteOperation} that manages the process of writing to the sink.
  *
  * <p>The process of writing to file-based sink is as follows:
  * <ol>
@@ -89,8 +89,8 @@
  *
  * <p>In order to ensure fault-tolerance, a bundle may be executed multiple times (e.g., in the
  * event of failure/retry or for redundancy). However, exactly one of these executions will have its
- * result passed to the finalize method. Each call to {@link FileBasedWriter#openWindowed}
- * or {@link FileBasedWriter#openUnwindowed} is passed a unique <i>bundle id</i> when it is called
+ * result passed to the finalize method. Each call to {@link Writer#openWindowed}
+ * or {@link Writer#openUnwindowed} is passed a unique <i>bundle id</i> when it is called
  * by the WriteFiles transform, so even redundant or retried bundles will have a unique way of
  * identifying
  * their output.
@@ -359,10 +359,10 @@ public final FilenamePolicy getFilenamePolicy() {
   public void validate(PipelineOptions options) {}
 
   /**
-   * Return a subclass of {@link FileBasedSink.FileBasedWriteOperation} that will manage the write
+   * Return a subclass of {@link WriteOperation} that will manage the write
    * to the sink.
    */
-  public abstract FileBasedWriteOperation<T> createWriteOperation();
+  public abstract WriteOperation<T> createWriteOperation();
 
   public void populateDisplayData(DisplayData.Builder builder) {
     getFilenamePolicy().populateDisplayData(builder);
@@ -371,24 +371,24 @@ public void populateDisplayData(DisplayData.Builder builder) {
   /**
    * Abstract operation that manages the process of writing to {@link FileBasedSink}.
    *
-   * <p>The primary responsibilities of the FileBasedWriteOperation is the management of output
-   * files. During a write, {@link FileBasedSink.FileBasedWriter}s write bundles to temporary file
+   * <p>The primary responsibilities of the WriteOperation is the management of output
+   * files. During a write, {@link Writer}s write bundles to temporary file
    * locations. After the bundles have been written,
    * <ol>
-   * <li>{@link FileBasedSink.FileBasedWriteOperation#finalize} is given a list of the temporary
+   * <li>{@link WriteOperation#finalize} is given a list of the temporary
    * files containing the output bundles.
    * <li>During finalize, these temporary files are copied to final output locations and named
    * according to a file naming template.
    * <li>Finally, any temporary files that were created during the write are removed.
    * </ol>
    *
-   * <p>Subclass implementations of FileBasedWriteOperation must implement
-   * {@link FileBasedSink.FileBasedWriteOperation#createWriter} to return a concrete
+   * <p>Subclass implementations of WriteOperation must implement
+   * {@link WriteOperation#createWriter} to return a concrete
    * FileBasedSinkWriter.
    *
    * <h2>Temporary and Output File Naming:</h2> During the write, bundles are written to temporary
    * files using the tempDirectory that can be provided via the constructor of
-   * FileBasedWriteOperation. These temporary files will be named
+   * WriteOperation. These temporary files will be named
    * {@code {tempDirectory}/{bundleId}}, where bundleId is the unique id of the bundle.
    * For example, if tempDirectory is "gs://my-bucket/my_temp_output", the output for a
    * bundle with bundle id 15723 will be "gs://my-bucket/my_temp_output/15723".
@@ -408,7 +408,7 @@ public void populateDisplayData(DisplayData.Builder builder) {
    *
    * @param <T> the type of values written to the sink.
    */
-  public abstract static class FileBasedWriteOperation<T> implements Serializable {
+  public abstract static class WriteOperation<T> implements Serializable {
     /**
      * The Sink that this WriteOperation will write to.
      */
@@ -427,7 +427,7 @@ protected static ResourceId buildTemporaryFilename(ResourceId tempDirectory, Str
     }
 
     /**
-     * Constructs a FileBasedWriteOperation using the default strategy for generating a temporary
+     * Constructs a WriteOperation using the default strategy for generating a temporary
      * directory from the base output filename.
      *
      * <p>Default is a uniquely named sibling of baseOutputFilename, e.g. if baseOutputFilename is
@@ -435,7 +435,7 @@ protected static ResourceId buildTemporaryFilename(ResourceId tempDirectory, Str
      *
      * @param sink the FileBasedSink that will be used to configure this write operation.
      */
-    public FileBasedWriteOperation(FileBasedSink<T> sink) {
+    public WriteOperation(FileBasedSink<T> sink) {
       this(sink, NestedValueProvider.of(
           sink.getBaseOutputDirectoryProvider(), new TemporaryDirectoryBuilder()));
     }
@@ -461,16 +461,16 @@ public ResourceId apply(ResourceId baseOutputDirectory) {
     }
 
     /**
-     * Create a new FileBasedWriteOperation.
+     * Create a new WriteOperation.
      *
      * @param sink the FileBasedSink that will be used to configure this write operation.
      * @param tempDirectory the base directory to be used for temporary output files.
      */
-    public FileBasedWriteOperation(FileBasedSink<T> sink, ResourceId tempDirectory) {
+    public WriteOperation(FileBasedSink<T> sink, ResourceId tempDirectory) {
       this(sink, StaticValueProvider.of(tempDirectory));
     }
 
-    private FileBasedWriteOperation(
+    private WriteOperation(
         FileBasedSink<T> sink, ValueProvider<ResourceId> tempDirectory) {
       this.sink = sink;
       this.tempDirectory = tempDirectory;
@@ -478,10 +478,10 @@ private FileBasedWriteOperation(
     }
 
     /**
-     * Clients must implement to return a subclass of {@link FileBasedSink.FileBasedWriter}. This
+     * Clients must implement to return a subclass of {@link Writer}. This
      * method must not mutate the state of the object.
      */
-    public abstract FileBasedWriter<T> createWriter() throws Exception;
+    public abstract Writer<T> createWriter() throws Exception;
 
     /**
      * Indicates that the operation will be performing windowed writes.
@@ -558,7 +558,7 @@ protected final Map<ResourceId, ResourceId> buildOutputFilenames(
     /**
      * Copy temporary files to final output filenames using the file naming template.
      *
-     * <p>Can be called from subclasses that override {@link FileBasedWriteOperation#finalize}.
+     * <p>Can be called from subclasses that override {@link WriteOperation#finalize}.
      *
      * <p>Files will be named according to the file naming template. The order of the output files
      * will be the same as the sorted order of the input filenames.  In other words, if the input
@@ -591,7 +591,7 @@ final void copyToOutputFiles(Map<ResourceId, ResourceId> filenames)
     /**
      * Removes temporary output files. Uses the temporary directory to find files to remove.
      *
-     * <p>Can be called from subclasses that override {@link FileBasedWriteOperation#finalize}.
+     * <p>Can be called from subclasses that override {@link WriteOperation#finalize}.
      * <b>Note:</b>If finalize is overridden and does <b>not</b> rename or otherwise finalize
      * temporary files, this method will remove them.
      */
@@ -667,15 +667,15 @@ protected final String getExtension() {
    * after the values in a bundle, respectively, as well as provide a MIME type for the output
    * channel.
    *
-   * <p>Multiple {@link FileBasedWriter} instances may be created on the same worker, and therefore
+   * <p>Multiple {@link Writer} instances may be created on the same worker, and therefore
    * any access to static members or methods should be thread safe.
    *
    * @param <T> the type of values to write.
    */
-  public abstract static class FileBasedWriter<T> {
-    private static final Logger LOG = LoggerFactory.getLogger(FileBasedWriter.class);
+  public abstract static class Writer<T> {
+    private static final Logger LOG = LoggerFactory.getLogger(Writer.class);
 
-    private final FileBasedWriteOperation<T> writeOperation;
+    private final WriteOperation<T> writeOperation;
 
     /** Unique id for this output bundle. */
     private String id;
@@ -704,9 +704,9 @@ public abstract static class FileBasedWriter<T> {
     private final String mimeType;
 
     /**
-     * Construct a new {@link FileBasedWriter} that will produce files of the given MIME type.
+     * Construct a new {@link Writer} that will produce files of the given MIME type.
      */
-    public FileBasedWriter(FileBasedWriteOperation<T> writeOperation, String mimeType) {
+    public Writer(WriteOperation<T> writeOperation, String mimeType) {
       checkNotNull(writeOperation);
       this.writeOperation = writeOperation;
       this.mimeType = mimeType;
@@ -739,7 +739,7 @@ protected void finishWrite() throws Exception {}
 
     /**
      *  Performs bundle initialization. For example, creates a temporary file for writing or
-     * initializes any state that will be used across calls to {@link FileBasedWriter#write}.
+     * initializes any state that will be used across calls to {@link Writer#write}.
      *
      * <p>The unique id that is given to open should be used to ensure that the writer's output
      * does not interfere with the output of other Writers, as a bundle may be executed many
@@ -818,7 +818,7 @@ private void open(String uId,
         closeChannelAndThrow(tempChannel, outputFile, e);
       }
 
-      // The caller shouldn't have to close() this FileBasedWriter if it fails to open(), so close
+      // The caller shouldn't have to close() this Writer if it fails to open(), so close
       // the channel if prepareWrite() or writeHeader() fails.
       String step = "";
       try {
@@ -895,9 +895,9 @@ public final FileResult close() throws Exception {
     }
 
     /**
-     * Return the FileBasedWriteOperation that this Writer belongs to.
+     * Return the WriteOperation that this Writer belongs to.
      */
-    public FileBasedWriteOperation<T> getWriteOperation() {
+    public WriteOperation<T> getWriteOperation() {
       return writeOperation;
     }
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
index 6350fb57aa98..f73d6f3d284f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
@@ -567,7 +567,7 @@ public ResourceId apply(ResourceId input) {
     }
 
     @Override
-    public FileBasedWriteOperation<byte[]> createWriteOperation() {
+    public WriteOperation<byte[]> createWriteOperation() {
       return new TFRecordWriteOperation(this);
     }
 
@@ -587,29 +587,29 @@ private static WritableByteChannelFactory writableByteChannelFactory(
     }
 
     /**
-     * A {@link org.apache.beam.sdk.io.FileBasedSink.FileBasedWriteOperation
-     * FileBasedWriteOperation} for TFRecord files.
+     * A {@link WriteOperation
+     * WriteOperation} for TFRecord files.
      */
-    private static class TFRecordWriteOperation extends FileBasedWriteOperation<byte[]> {
+    private static class TFRecordWriteOperation extends WriteOperation<byte[]> {
       private TFRecordWriteOperation(TFRecordSink sink) {
         super(sink);
       }
 
       @Override
-      public FileBasedWriter<byte[]> createWriter() throws Exception {
+      public Writer<byte[]> createWriter() throws Exception {
         return new TFRecordWriter(this);
       }
     }
 
     /**
-     * A {@link org.apache.beam.sdk.io.FileBasedSink.FileBasedWriter FileBasedWriter}
+     * A {@link Writer Writer}
      * for TFRecord files.
      */
-    private static class TFRecordWriter extends FileBasedWriter<byte[]> {
+    private static class TFRecordWriter extends Writer<byte[]> {
       private WritableByteChannel outChannel;
       private TFRecordCodec codec;
 
-      private TFRecordWriter(FileBasedWriteOperation<byte[]> writeOperation) {
+      private TFRecordWriter(WriteOperation<byte[]> writeOperation) {
         super(writeOperation, MimeTypes.BINARY);
       }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java
index 0ba537e554a7..511d6976a4e9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSink.java
@@ -49,12 +49,12 @@ class TextSink extends FileBasedSink<String> {
     this.footer = footer;
   }
   @Override
-  public FileBasedWriteOperation<String> createWriteOperation() {
+  public WriteOperation<String> createWriteOperation() {
     return new TextWriteOperation(this, header, footer);
   }
 
-  /** A {@link FileBasedWriteOperation FileBasedWriteOperation} for text files. */
-  private static class TextWriteOperation extends FileBasedWriteOperation<String> {
+  /** A {@link WriteOperation WriteOperation} for text files. */
+  private static class TextWriteOperation extends WriteOperation<String> {
     @Nullable private final String header;
     @Nullable private final String footer;
 
@@ -65,20 +65,20 @@ private TextWriteOperation(TextSink sink, @Nullable String header, @Nullable Str
     }
 
     @Override
-    public FileBasedWriter<String> createWriter() throws Exception {
+    public Writer<String> createWriter() throws Exception {
       return new TextWriter(this, header, footer);
     }
   }
 
-  /** A {@link FileBasedWriter FileBasedWriter} for text files. */
-  private static class TextWriter extends FileBasedWriter<String> {
+  /** A {@link Writer Writer} for text files. */
+  private static class TextWriter extends Writer<String> {
     private static final String NEWLINE = "\n";
     @Nullable private final String header;
     @Nullable private final String footer;
     private OutputStreamWriter out;
 
     public TextWriter(
-        FileBasedWriteOperation<String> writeOperation,
+        WriteOperation<String> writeOperation,
         @Nullable String header,
         @Nullable String footer) {
       super(writeOperation, MimeTypes.TEXT);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
index 5b219027e64c..af24a8f13377 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
@@ -30,10 +30,10 @@
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.FileBasedSink.FileBasedWriteOperation;
-import org.apache.beam.sdk.io.FileBasedSink.FileBasedWriter;
 import org.apache.beam.sdk.io.FileBasedSink.FileResult;
 import org.apache.beam.sdk.io.FileBasedSink.FileResultCoder;
+import org.apache.beam.sdk.io.FileBasedSink.WriteOperation;
+import org.apache.beam.sdk.io.FileBasedSink.Writer;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
@@ -63,7 +63,7 @@
  * finalization of the write. The output of a write is {@link PDone}.
  *
  * <p>By default, every bundle in the input {@link PCollection} will be processed by a
- * {@link org.apache.beam.sdk.io.FileBasedSink.FileBasedWriteOperation}, so the number of output
+ * {@link WriteOperation}, so the number of output
  * will vary based on runner behavior, though at least 1 output will always be produced. The
  * exact parallelism of the write stage can be controlled using {@link WriteFiles#withNumShards},
  * typically used to control how many files are produced or to globally limit the number of
@@ -86,7 +86,7 @@ public class WriteFiles<T> extends PTransform<PCollection<T>, PDone> {
   private static final int UNKNOWN_NUMSHARDS = -1;
 
   private FileBasedSink<T> sink;
-  private FileBasedWriteOperation<T> writeOperation;
+  private WriteOperation<T> writeOperation;
   // This allows the number of shards to be dynamically computed based on the input
   // PCollection.
   @Nullable
@@ -238,13 +238,13 @@ public WriteFiles<T> withWindowedWrites() {
   }
 
   /**
-   * Writes all the elements in a bundle using a {@link FileBasedWriter} produced by the
-   * {@link FileBasedSink.FileBasedWriteOperation} associated with the {@link FileBasedSink}.
+   * Writes all the elements in a bundle using a {@link Writer} produced by the
+   * {@link WriteOperation} associated with the {@link FileBasedSink}.
    */
   private class WriteBundles extends DoFn<T, FileResult> {
     // Writer that will write the records in this bundle. Lazily
     // initialized in processElement.
-    private FileBasedWriter<T> writer = null;
+    private Writer<T> writer = null;
     private BoundedWindow window = null;
 
     WriteBundles() {
@@ -320,7 +320,7 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
       // In a sharded write, single input element represents one shard. We can open and close
       // the writer in each call to processElement.
       LOG.info("Opening writer for write operation {}", writeOperation);
-      FileBasedWriter<T> writer = writeOperation.createWriter();
+      Writer<T> writer = writeOperation.createWriter();
       if (windowedWrites) {
         writer.openWindowed(UUID.randomUUID().toString(), window, c.pane(), c.element().getKey(),
             numShards);
@@ -404,29 +404,29 @@ public void processElement(ProcessContext context) {
   /**
    * A write is performed as sequence of three {@link ParDo}'s.
    *
-   * <p>This singleton collection containing the FileBasedWriteOperation is then used as a side
+   * <p>This singleton collection containing the WriteOperation is then used as a side
    * input to a ParDo over the PCollection of elements to write. In this bundle-writing phase,
-   * {@link FileBasedWriteOperation#createWriter} is called to obtain a {@link FileBasedWriter}.
-   * {@link FileBasedWriter#open} and {@link FileBasedWriter#close} are called in
+   * {@link WriteOperation#createWriter} is called to obtain a {@link Writer}.
+   * {@link Writer#open} and {@link Writer#close} are called in
    * {@link DoFn.StartBundle} and {@link DoFn.FinishBundle}, respectively, and
-   * {@link FileBasedWriter#write} method is called for every element in the bundle. The output
+   * {@link Writer#write} method is called for every element in the bundle. The output
    * of this ParDo is a PCollection of <i>writer result</i> objects (see {@link FileBasedSink}
    * for a description of writer results)-one for each bundle.
    *
    * <p>The final do-once ParDo uses a singleton collection asinput and the collection of writer
-   * results as a side-input. In this ParDo, {@link FileBasedWriteOperation#finalize} is called
+   * results as a side-input. In this ParDo, {@link WriteOperation#finalize} is called
    * to finalize the write.
    *
-   * <p>If the write of any element in the PCollection fails, {@link FileBasedWriter#close} will be
+   * <p>If the write of any element in the PCollection fails, {@link Writer#close} will be
    * called before the exception that caused the write to fail is propagated and the write result
    * will be discarded.
    *
-   * <p>Since the {@link FileBasedWriteOperation} is serialized after the initialization ParDo and
+   * <p>Since the {@link WriteOperation} is serialized after the initialization ParDo and
    * deserialized in the bundle-writing and finalization phases, any state change to the
-   * FileBasedWriteOperation object that occurs during initialization is visible in the latter
-   * phases. However, the FileBasedWriteOperation is not serialized after the bundle-writing
+   * WriteOperation object that occurs during initialization is visible in the latter
+   * phases. However, the WriteOperation is not serialized after the bundle-writing
    * phase. This is why implementations should guarantee that
-   * {@link FileBasedWriteOperation#createWriter} does not mutate FileBasedWriteOperation).
+   * {@link WriteOperation#createWriter} does not mutate WriteOperation).
    */
   private PDone createWrite(PCollection<T> input) {
     Pipeline p = input.getPipeline();
@@ -442,9 +442,9 @@ private PDone createWrite(PCollection<T> input) {
 
 
     // Perform the per-bundle writes as a ParDo on the input PCollection (with the
-    // FileBasedWriteOperation as a side input) and collect the results of the writes in a
+    // WriteOperation as a side input) and collect the results of the writes in a
     // PCollection. There is a dependency between this ParDo and the first (the
-    // FileBasedWriteOperation PCollection as a side input), so this will happen after the
+    // WriteOperation PCollection as a side input), so this will happen after the
     // initial ParDo.
     PCollection<FileResult> results;
     final PCollectionView<Integer> numShardsView;
@@ -511,7 +511,7 @@ public void processElement(ProcessContext c) throws Exception {
 
       // Finalize the write in another do-once ParDo on the singleton collection containing the
       // Writer. The results from the per-bundle writes are given as an Iterable side input.
-      // The FileBasedWriteOperation's state is the same as after its initialization in the first
+      // The WriteOperation's state is the same as after its initialization in the first
       // do-once ParDo. There is a dependency between this ParDo and the parallel write (the writer
       // results collection as a side input), so it will happen after the parallel write.
       // For the non-windowed case, we guarantee that  if no data is written but the user has
@@ -542,7 +542,7 @@ public void processElement(ProcessContext c) throws Exception {
                     "Creating {} empty output shards in addition to {} written for a total of {}.",
                     extraShardsNeeded, results.size(), minShardsNeeded);
                 for (int i = 0; i < extraShardsNeeded; ++i) {
-                  FileBasedWriter<T> writer = writeOperation.createWriter();
+                  Writer<T> writer = writeOperation.createWriter();
                   writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM,
                       UNKNOWN_NUMSHARDS);
                   FileResult emptyWrite = writer.close();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
index d9bcef441d43..c3f2a589b739 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
@@ -46,12 +46,12 @@
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
 import org.apache.beam.sdk.io.FileBasedSink.CompressionType;
-import org.apache.beam.sdk.io.FileBasedSink.FileBasedWriteOperation;
-import org.apache.beam.sdk.io.FileBasedSink.FileBasedWriter;
 import org.apache.beam.sdk.io.FileBasedSink.FileResult;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy.Context;
 import org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory;
+import org.apache.beam.sdk.io.FileBasedSink.WriteOperation;
+import org.apache.beam.sdk.io.FileBasedSink.Writer;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -88,7 +88,7 @@ private ResourceId getBaseTempDirectory() {
   }
 
   /**
-   * FileBasedWriter opens the correct file, writes the header, footer, and elements in the
+   * Writer opens the correct file, writes the header, footer, and elements in the
    * correct order, and returns the correct filename.
    */
   @Test
@@ -187,13 +187,13 @@ public void testFinalizeWithIntermediateState() throws Exception {
   }
 
   /**
-   * Generate n temporary files using the temporary file pattern of FileBasedWriter.
+   * Generate n temporary files using the temporary file pattern of Writer.
    */
   private List<File> generateTemporaryFilesForFinalize(int numFiles) throws Exception {
     List<File> temporaryFiles = new ArrayList<>();
     for (int i = 0; i < numFiles; i++) {
       ResourceId temporaryFile =
-          FileBasedWriteOperation.buildTemporaryFilename(getBaseTempDirectory(), "" + i);
+          WriteOperation.buildTemporaryFilename(getBaseTempDirectory(), "" + i);
       File tmpFile = new File(tmpFolder.getRoot(), temporaryFile.toString());
       tmpFile.getParentFile().mkdirs();
       assertTrue(tmpFile.createNewFile());
@@ -242,14 +242,14 @@ private void testRemoveTemporaryFiles(int numFiles, ResourceId tempDirectory)
     SimpleSink sink =
         new SimpleSink(getBaseOutputDirectory(), prefix, "", "");
 
-    FileBasedWriteOperation<String> writeOp =
+    WriteOperation<String> writeOp =
         new SimpleSink.SimpleWriteOperation(sink, tempDirectory);
 
     List<File> temporaryFiles = new ArrayList<>();
     List<File> outputFiles = new ArrayList<>();
     for (int i = 0; i < numFiles; i++) {
       ResourceId tempResource =
-          FileBasedWriteOperation.buildTemporaryFilename(tempDirectory, prefix + i);
+          WriteOperation.buildTemporaryFilename(tempDirectory, prefix + i);
       File tmpFile = new File(tempResource.toString());
       tmpFile.getParentFile().mkdirs();
       assertTrue("not able to create new temp file", tmpFile.createNewFile());
@@ -487,17 +487,17 @@ private File writeValuesWithWritableByteChannelFactory(final WritableByteChannel
   }
 
   /**
-   * {@link FileBasedWriter} writes to the {@link WritableByteChannel} provided by
+   * {@link Writer} writes to the {@link WritableByteChannel} provided by
    * {@link DrunkWritableByteChannelFactory}.
    */
   @Test
   public void testFileBasedWriterWithWritableByteChannelFactory() throws Exception {
     final String testUid = "testId";
     ResourceId root = getBaseOutputDirectory();
-    FileBasedWriteOperation<String> writeOp =
+    WriteOperation<String> writeOp =
         new SimpleSink(root, "file", "-SS-of-NN", "txt", new DrunkWritableByteChannelFactory())
             .createWriteOperation();
-    final FileBasedWriter<String> writer = writeOp.createWriter();
+    final Writer<String> writer = writeOp.createWriter();
     final ResourceId expectedFile =
         writeOp.tempDirectory.get().resolve(testUid, StandardResolveOptions.RESOLVE_FILE);
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
index 9265520b2486..c97313d397dd 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
@@ -45,7 +45,7 @@ public SimpleWriteOperation createWriteOperation() {
     return new SimpleWriteOperation(this);
   }
 
-  static final class SimpleWriteOperation extends FileBasedWriteOperation<String> {
+  static final class SimpleWriteOperation extends WriteOperation<String> {
     public SimpleWriteOperation(SimpleSink sink, ResourceId tempOutputDirectory) {
       super(sink, tempOutputDirectory);
     }
@@ -60,7 +60,7 @@ public SimpleWriter createWriter() throws Exception {
     }
   }
 
-  static final class SimpleWriter extends FileBasedWriter<String> {
+  static final class SimpleWriter extends Writer<String> {
     static final String HEADER = "header";
     static final String FOOTER = "footer";
 
diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java
index 963ab1b10884..60075a777221 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java
@@ -76,9 +76,9 @@ void populateFileBasedDisplayData(DisplayData.Builder builder) {
   }
 
   /**
-   * {@link FileBasedSink.FileBasedWriteOperation} for XML {@link FileBasedSink}s.
+   * {@link WriteOperation} for XML {@link FileBasedSink}s.
    */
-  protected static final class XmlWriteOperation<T> extends FileBasedWriteOperation<T> {
+  protected static final class XmlWriteOperation<T> extends WriteOperation<T> {
     public XmlWriteOperation(XmlSink<T> sink) {
       super(sink);
     }
@@ -113,9 +113,9 @@ ResourceId getTemporaryDirectory() {
   }
 
   /**
-   * A {@link FileBasedWriter} that can write objects as XML elements.
+   * A {@link Writer} that can write objects as XML elements.
    */
-  protected static final class XmlWriter<T> extends FileBasedWriter<T> {
+  protected static final class XmlWriter<T> extends Writer<T> {
     final Marshaller marshaller;
     private OutputStream os = null;
 

From 2d5fbf6a3d9df29c9cd3a96caabdb396f1857e75 Mon Sep 17 00:00:00 2001
From: Reuven Lax <relax@google.com>
Date: Wed, 5 Apr 2017 12:13:44 -0700
Subject: [PATCH 176/387] Implement dynamic-sharding for windowed file outputs,
 and add an integration test.

---
 .../beam/examples/WindowedWordCount.java      |   7 +-
 .../common/WriteOneFilePerWindow.java         |  24 +-
 .../beam/examples/WindowedWordCountIT.java    |  23 +-
 ...ltCoderCloudObjectTranslatorRegistrar.java |   2 -
 .../org/apache/beam/sdk/io/FileBasedSink.java | 225 +++++++++++-------
 .../org/apache/beam/sdk/io/WriteFiles.java    | 144 ++++++-----
 .../apache/beam/sdk/io/FileBasedSinkTest.java | 148 ++++++------
 7 files changed, 332 insertions(+), 241 deletions(-)

diff --git a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
index 45746af3efd1..20b48e435929 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
@@ -161,12 +161,15 @@ public interface Options extends WordCount.WordCountOptions,
     @Default.InstanceFactory(DefaultToMinTimestampPlusOneHour.class)
     Long getMaxTimestampMillis();
     void setMaxTimestampMillis(Long value);
+
+    @Description("Fixed number of shards to produce per window, or null for runner-chosen sharding")
+    Integer getNumShards();
+    void setNumShards(Integer numShards);
   }
 
   public static void main(String[] args) throws IOException {
     Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
     final String output = options.getOutput();
-    final Duration windowSize = Duration.standardMinutes(options.getWindowSize());
     final Instant minTimestamp = new Instant(options.getMinTimestampMillis());
     final Instant maxTimestamp = new Instant(options.getMaxTimestampMillis());
 
@@ -207,7 +210,7 @@ public static void main(String[] args) throws IOException {
      */
     wordCounts
         .apply(MapElements.via(new WordCount.FormatAsTextFn()))
-        .apply(new WriteOneFilePerWindow(output));
+        .apply(new WriteOneFilePerWindow(output, options.getNumShards()));
 
     PipelineResult result = pipeline.run();
     try {
diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java b/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java
index ed35c8a05503..5e6df9c781fd 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java
@@ -19,6 +19,7 @@
 
 import static com.google.common.base.Verify.verifyNotNull;
 
+import javax.annotation.Nullable;
 import org.apache.beam.sdk.io.FileBasedSink;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
 import org.apache.beam.sdk.io.TextIO;
@@ -40,12 +41,14 @@
  * lessons.
  */
 public class WriteOneFilePerWindow extends PTransform<PCollection<String>, PDone> {
-
   private static final DateTimeFormatter FORMATTER = ISODateTimeFormat.hourMinute();
-  private final String filenamePrefix;
+  private String filenamePrefix;
+  @Nullable
+  private Integer numShards;
 
-  public WriteOneFilePerWindow(String filenamePrefix) {
+  public WriteOneFilePerWindow(String filenamePrefix, Integer numShards) {
     this.filenamePrefix = filenamePrefix;
+    this.numShards = numShards;
   }
 
   @Override
@@ -61,12 +64,15 @@ public PDone expand(PCollection<String> input) {
           resource);
     }
 
-    return input.apply(
-        TextIO.write()
-            .to(resource.getCurrentDirectory())
-            .withFilenamePolicy(new PerWindowFiles(prefix))
-            .withWindowedWrites()
-            .withNumShards(3));
+
+    TextIO.Write write = TextIO.write()
+        .to(resource.getCurrentDirectory())
+        .withFilenamePolicy(new PerWindowFiles(prefix))
+        .withWindowedWrites();
+    if (numShards != null) {
+      write = write.withNumShards(numShards);
+    }
+    return input.apply(write);
   }
 
   /**
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index 93c4543e0739..eb7e4c4e936f 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -86,14 +86,29 @@ public static void setUp() {
   }
 
   @Test
-  public void testWindowedWordCountInBatch() throws Exception {
-    testWindowedWordCountPipeline(defaultOptions());
+  public void testWindowedWordCountInBatchDynamicSharding() throws Exception {
+    WindowedWordCountITOptions options = batchOptions();
+    // This is the default value, but make it explicit
+    options.setNumShards(null);
+    testWindowedWordCountPipeline(options);
   }
 
+  @Test
+  public void testWindowedWordCountInBatchStaticSharding() throws Exception {
+    WindowedWordCountITOptions options = batchOptions();
+    options.setNumShards(3);
+    testWindowedWordCountPipeline(options);
+  }
+
+  // TODO: add a test with streaming and dynamic sharding after resolving
+  // https://issues.apache.org/jira/browse/BEAM-1438
+
   @Test
   @Category(StreamingIT.class)
-  public void testWindowedWordCountInStreaming() throws Exception {
-    testWindowedWordCountPipeline(streamingOptions());
+  public void testWindowedWordCountInStreamingStaticSharding() throws Exception {
+    WindowedWordCountITOptions options = streamingOptions();
+    options.setNumShards(3);
+    testWindowedWordCountPipeline(options);
   }
 
   private WindowedWordCountITOptions defaultOptions() throws Exception {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java
index 45670988be15..5d42a5fedde0 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java
@@ -47,7 +47,6 @@
 import org.apache.beam.sdk.coders.TextualIntegerCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.FileBasedSink.FileResultCoder;
 import org.apache.beam.sdk.io.gcp.bigquery.TableDestinationCoder;
 import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder;
 
@@ -91,7 +90,6 @@ public class DefaultCoderCloudObjectTranslatorRegistrar
           ByteCoder.class,
           DoubleCoder.class,
           DurationCoder.class,
-          FileResultCoder.class,
           FooterCoder.class,
           InstantCoder.class,
           IsmShardCoder.class,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index c2e230d47e60..21177941d8f5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -34,7 +34,9 @@
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -43,10 +45,12 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.GZIPOutputStream;
 import javax.annotation.Nullable;
-import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.NullableCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy.Context;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy.WindowedContext;
 import org.apache.beam.sdk.io.fs.MatchResult;
@@ -63,6 +67,7 @@
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo.PaneInfoCoder;
 import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
@@ -306,8 +311,7 @@ public void populateDisplayData(DisplayData.Builder builder) {
   }
 
   /** The policy used to generate names of files to be produced. */
-  @VisibleForTesting
-  final FilenamePolicy filenamePolicy;
+  private final FilenamePolicy filenamePolicy;
   /** The directory to which files will be written. */
   private final ValueProvider<ResourceId> baseOutputDirectoryProvider;
 
@@ -523,27 +527,43 @@ public void finalize(Iterable<FileResult> writerResults) throws Exception {
 
     protected final Map<ResourceId, ResourceId> buildOutputFilenames(
         Iterable<FileResult> writerResults) {
+      int numShards = Iterables.size(writerResults);
       Map<ResourceId, ResourceId> outputFilenames = new HashMap<>();
-      List<ResourceId> files = new ArrayList<>();
+
+      List<FileResult> unshardedFiles = new ArrayList<>();
+      FilenamePolicy policy = getSink().getFilenamePolicy();
       for (FileResult result : writerResults) {
-        if (result.getDestinationFilename() != null) {
-          outputFilenames.put(result.getFilename(), result.getDestinationFilename());
+        if (result.getShard() != WriteFiles.UNKNOWN_SHARDNUM) {
+          outputFilenames.put(result.getTempFilename(),
+              result.getDestinationFile(policy, getSink().getBaseOutputDirectoryProvider().get(),
+                  numShards, getSink().getExtension()));
         } else {
-          files.add(result.getFilename());
+          unshardedFiles.add(result);
         }
       }
 
       // writerResults won't contain destination filenames, so we dynamically generate them here.
-      if (files.size() > 0) {
+      if (unshardedFiles.size() > 0) {
         checkArgument(outputFilenames.isEmpty());
-        // Sort files for idempotence.
-        files = Ordering.usingToString().sortedCopy(files);
-        ResourceId outputDirectory = getSink().getBaseOutputDirectoryProvider().get();
-        FilenamePolicy filenamePolicy = getSink().filenamePolicy;
-        for (int i = 0; i < files.size(); i++) {
-          outputFilenames.put(files.get(i),
-              filenamePolicy.unwindowedFilename(outputDirectory, new Context(i, files.size()),
-                  getSink().getExtension()));
+
+        // Sort files for idempotence. Sort by temporary filename.
+        // Note that this codepath should not be used when processing triggered windows. In the
+        // case of triggers, the list of FileResult objects in the Finalize iterable is not
+        // deterministic, and might change over retries. This breaks the assumption below that
+        // sorting the FileResult objects provides idempotency.
+        unshardedFiles = Ordering.from(new Comparator<FileResult>() {
+          @Override
+          public int compare(FileResult first, FileResult second) {
+            return first.getTempFilename().toString().compareTo(
+                second.getTempFilename().toString());
+          }
+        }).sortedCopy(unshardedFiles);
+        for (int i = 0; i < unshardedFiles.size(); i++) {
+          FileResult result = unshardedFiles.get(i);
+          result.setShard(i);
+          outputFilenames.put(result.getTempFilename(),
+              result.getDestinationFile(policy, getSink().getBaseOutputDirectoryProvider().get(),
+                  numShards, getSink().getExtension()));
         }
       }
 
@@ -647,6 +667,19 @@ final void removeTemporaryFiles(
     public FileBasedSink<T> getSink() {
       return sink;
     }
+
+    @Override
+    public String toString() {
+      String tempDirectoryStr =
+          tempDirectory.isAccessible() ? tempDirectory.get().toString() : tempDirectory.toString();
+      return getClass().getSimpleName()
+          + "{"
+          + "tempDirectory="
+          + tempDirectoryStr
+          + ", windowedWrites="
+          + windowedWrites
+          + '}';
+    }
   }
 
   /** Returns the extension that will be written to the produced files. */
@@ -683,7 +716,6 @@ public abstract static class Writer<T> {
     private BoundedWindow window;
     private PaneInfo paneInfo;
     private int shard = -1;
-    private int numShards = -1;
 
     /** The output file for this bundle. May be null if opening failed. */
     private @Nullable ResourceId outputFile;
@@ -738,24 +770,23 @@ protected void writeFooter() throws Exception {}
     protected void finishWrite() throws Exception {}
 
     /**
-     *  Performs bundle initialization. For example, creates a temporary file for writing or
+     * Performs bundle initialization. For example, creates a temporary file for writing or
      * initializes any state that will be used across calls to {@link Writer#write}.
      *
-     * <p>The unique id that is given to open should be used to ensure that the writer's output
-     * does not interfere with the output of other Writers, as a bundle may be executed many
-     * times for fault tolerance.
+     * <p>The unique id that is given to open should be used to ensure that the writer's output does
+     * not interfere with the output of other Writers, as a bundle may be executed many times for
+     * fault tolerance.
      *
-     * <p>The window and paneInfo arguments are populated when windowed writes are requested.
-     * shard and numShards are populated for the case of static sharding. In cases where the
-     * runner is dynamically picking sharding, shard and numShards might both be set to -1.
+     * <p>The window and paneInfo arguments are populated when windowed writes are requested. shard
+     * id populated for the case of static sharding. In cases where the runner is dynamically
+     * picking sharding, shard might be set to -1.
      */
-    public final void openWindowed(
-        String uId, BoundedWindow window, PaneInfo paneInfo, int shard, int numShards)
+    public final void openWindowed(String uId, BoundedWindow window, PaneInfo paneInfo, int shard)
         throws Exception {
       if (!getWriteOperation().windowedWrites) {
         throw new IllegalStateException("openWindowed called a non-windowed sink.");
       }
-      open(uId, window, paneInfo, shard, numShards);
+      open(uId, window, paneInfo, shard);
     }
 
     /**
@@ -767,13 +798,11 @@ public final void openWindowed(
      * Similar to {@link #openWindowed} however for the case where unwindowed writes were
      * requested.
      */
-    public final void openUnwindowed(String uId,
-                                     int shard,
-                                     int numShards) throws Exception {
+    public final void openUnwindowed(String uId, int shard) throws Exception {
       if (getWriteOperation().windowedWrites) {
         throw new IllegalStateException("openUnwindowed called a windowed sink.");
       }
-      open(uId, null, null, shard, numShards);
+      open(uId, null, null, shard);
     }
 
     // Helper function to close a channel, on exception cases.
@@ -792,13 +821,11 @@ private static void closeChannelAndThrow(
     private void open(String uId,
                       @Nullable BoundedWindow window,
                       @Nullable PaneInfo paneInfo,
-                      int shard,
-                      int numShards) throws Exception {
+                      int shard) throws Exception {
       this.id = uId;
       this.window = window;
       this.paneInfo = paneInfo;
       this.shard = shard;
-      this.numShards = numShards;
       ResourceId tempDirectory = getWriteOperation().tempDirectory.get();
       outputFile = tempDirectory.resolve(id, StandardResolveOptions.RESOLVE_FILE);
       verifyNotNull(
@@ -874,23 +901,8 @@ public final FileResult close() throws Exception {
         throw new IOException(String.format("Failed closing channel to %s", outputFile), e);
       }
 
-      FileBasedSink<T> sink = getWriteOperation().getSink();
-      ResourceId outputDirectory = sink.getBaseOutputDirectoryProvider().get();
-      FilenamePolicy filenamePolicy = sink.filenamePolicy;
-      String extension = sink.getExtension();
-      @Nullable ResourceId destinationFile;
-      if (window != null) {
-        destinationFile = filenamePolicy.windowedFilename(outputDirectory, new WindowedContext(
-            window, paneInfo, shard, numShards), extension);
-      } else if (numShards > 0) {
-        destinationFile = filenamePolicy.unwindowedFilename(
-            outputDirectory, new Context(shard, numShards), extension);
-      } else {
-        // Destination filename to be generated in the next step.
-        destinationFile = null;
-      }
-      FileResult result = new FileResult(outputFile, destinationFile);
-      LOG.debug("Result for bundle {}: {} {}", this.id, outputFile, destinationFile);
+      FileResult result = new FileResult(outputFile, shard, window, paneInfo);
+      LOG.debug("Result for bundle {}: {}", this.id, outputFile);
       return result;
     }
 
@@ -906,31 +918,56 @@ public WriteOperation<T> getWriteOperation() {
    * Result of a single bundle write. Contains the filename produced by the bundle, and if known
    * the final output filename.
    */
-  public static final class FileResult implements Serializable {
-    private final ResourceId filename;
-    @Nullable private final ResourceId destinationFilename;
+  public static final class FileResult {
+    private final ResourceId tempFilename;
+    private int shard;
+    private BoundedWindow window;
+    private PaneInfo paneInfo;
 
-    public FileResult(ResourceId filename, @Nullable ResourceId destinationFilename) {
-      this.filename = filename;
-      this.destinationFilename = destinationFilename;
+    public FileResult(ResourceId tempFilename, int shard, BoundedWindow window,
+                      PaneInfo paneInfo) {
+      this.tempFilename = tempFilename;
+      this.shard = shard;
+      this.window = window;
+      this.paneInfo = paneInfo;
     }
 
-    public ResourceId getFilename() {
-      return filename;
+    public ResourceId getTempFilename() {
+      return tempFilename;
     }
 
-    /**
-     * The filename to be written. Will be null if the output filename is unknown because the number
-     * of shards is determined dynamically by the runner.
-     */
-    @Nullable public ResourceId getDestinationFilename() {
-      return destinationFilename;
+    public int getShard() {
+      return shard;
+    }
+
+    public void setShard(int shard) {
+      this.shard = shard;
+    }
+
+    public BoundedWindow getWindow() {
+      return window;
+    }
+
+    public PaneInfo getPaneInfo() {
+      return paneInfo;
+    }
+
+    public ResourceId getDestinationFile(FilenamePolicy policy, ResourceId outputDirectory,
+                                         int numShards, String extension) {
+      checkArgument(getShard() != WriteFiles.UNKNOWN_SHARDNUM);
+      checkArgument(numShards > 0);
+      if (getWindow() != null) {
+        return policy.windowedFilename(outputDirectory, new WindowedContext(
+            getWindow(), getPaneInfo(), getShard(), numShards), extension);
+      } else {
+        return policy.unwindowedFilename(outputDirectory, new Context(getShard(), numShards),
+            extension);
+      }
     }
 
     public String toString() {
-      return MoreObjects.toStringHelper(FileResult.class)
-          .add("filename", filename)
-          .add("destinationFilename", destinationFilename)
+      return MoreObjects.toStringHelper(FileBasedSink.FileResult.class)
+          .add("tempFilename", tempFilename)
           .toString();
     }
   }
@@ -938,12 +975,23 @@ public String toString() {
   /**
    * A coder for {@link FileResult} objects.
    */
-  public static final class FileResultCoder extends AtomicCoder<FileResult> {
-    private static final FileResultCoder INSTANCE = new FileResultCoder();
-    private final NullableCoder<String> stringCoder = NullableCoder.of(StringUtf8Coder.of());
+  public static final class FileResultCoder extends StructuredCoder<FileResult> {
+    private final Coder<String> stringCoder = StringUtf8Coder.of();
+    private final Coder<Integer> integerCoder = VarIntCoder.of();
+    private final Coder<PaneInfo> paneInfoCoder = NullableCoder.of(PaneInfoCoder.INSTANCE);
+    private final Coder<BoundedWindow> windowCoder;
+
+    protected FileResultCoder(Coder<BoundedWindow> windowCoder) {
+      this.windowCoder = NullableCoder.of(windowCoder);
+    }
+
+    public static FileResultCoder of(Coder<BoundedWindow> windowCoder) {
+      return new FileResultCoder(windowCoder);
+    }
 
-    public static FileResultCoder of() {
-      return INSTANCE;
+    @Override
+    public List<? extends Coder<?>> getCoderArguments() {
+      return Arrays.asList(windowCoder);
     }
 
     @Override
@@ -952,29 +1000,30 @@ public void encode(FileResult value, OutputStream outStream)
       if (value == null) {
         throw new CoderException("cannot encode a null value");
       }
-      stringCoder.encode(value.getFilename().toString(), outStream);
-      if (value.getDestinationFilename() == null) {
-        stringCoder.encode(null, outStream);
-      } else {
-        stringCoder.encode(value.getDestinationFilename().toString(), outStream);
-      }
+      stringCoder.encode(value.getTempFilename().toString(), outStream);
+      windowCoder.encode(value.getWindow(), outStream);
+      paneInfoCoder.encode(value.getPaneInfo(), outStream);
+      integerCoder.encode(value.getShard(), outStream);
     }
 
     @Override
-    public FileResult decode(InputStream inStream) throws IOException {
-      String filename = stringCoder.decode(inStream);
-      assert filename != null;  // fixes a compiler warning
-      @Nullable String destinationFilename = stringCoder.decode(inStream);
-      return new FileResult(
-          FileSystems.matchNewResource(filename, false /* isDirectory */),
-          destinationFilename == null
-              ? null
-              : FileSystems.matchNewResource(destinationFilename, false /* isDirectory */));
+    public FileResult decode(InputStream inStream)
+        throws IOException {
+      String tempFilename = stringCoder.decode(inStream);
+      assert tempFilename != null;  // fixes a compiler warning
+      BoundedWindow window = windowCoder.decode(inStream);
+      PaneInfo paneInfo = paneInfoCoder.decode(inStream);
+      int shard = integerCoder.decode(inStream);
+      return new FileResult(FileSystems.matchNewResource(tempFilename, false /* isDirectory */),
+          shard, window, paneInfo);
     }
 
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
       stringCoder.verifyDeterministic();
+      windowCoder.verifyDeterministic();
+      paneInfoCoder.verifyDeterministic();
+      integerCoder.verifyDeterministic();
     }
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
index af24a8f13377..cab58621be38 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
@@ -22,12 +22,15 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.FileBasedSink.FileResult;
@@ -48,6 +51,7 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
@@ -82,9 +86,7 @@
 public class WriteFiles<T> extends PTransform<PCollection<T>, PDone> {
   private static final Logger LOG = LoggerFactory.getLogger(WriteFiles.class);
 
-  private static final int UNKNOWN_SHARDNUM = -1;
-  private static final int UNKNOWN_NUMSHARDS = -1;
-
+  static final int UNKNOWN_SHARDNUM = -1;
   private FileBasedSink<T> sink;
   private WriteOperation<T> writeOperation;
   // This allows the number of shards to be dynamically computed based on the input
@@ -119,9 +121,18 @@ private WriteFiles(
 
   @Override
   public PDone expand(PCollection<T> input) {
-    checkArgument(IsBounded.BOUNDED == input.isBounded() || windowedWrites,
-        "%s can only be applied to an unbounded PCollection if doing windowed writes",
-        WriteFiles.class.getSimpleName());
+    if (input.isBounded() == IsBounded.UNBOUNDED) {
+      checkArgument(windowedWrites,
+          "Must use windowed writes when applying %s to an unbounded PCollection",
+          WriteFiles.class.getSimpleName());
+      // The reason for this is https://issues.apache.org/jira/browse/BEAM-1438
+      // and similar behavior in other runners.
+      checkArgument(
+          computeNumShards != null || numShardsProvider != null,
+          "When applying %s to an unbounded PCollection, "
+              + "must specify number of output shards explicitly",
+          WriteFiles.class.getSimpleName());
+    }
     this.writeOperation = sink.createWriteOperation();
     this.writeOperation.setWindowedWrites(windowedWrites);
     return createWrite(input);
@@ -246,26 +257,50 @@ private class WriteBundles extends DoFn<T, FileResult> {
     // initialized in processElement.
     private Writer<T> writer = null;
     private BoundedWindow window = null;
+    private Map<KV<BoundedWindow, PaneInfo>, Writer<T>> windowedWriters;
 
-    WriteBundles() {
-    }
-
-    @ProcessElement
-    public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
-      // Lazily initialize the Writer
-      if (writer == null) {
-        LOG.info("Opening writer for write operation {}", writeOperation);
-        writer = writeOperation.createWriter();
-
-        if (windowedWrites) {
-          writer.openWindowed(UUID.randomUUID().toString(), window, c.pane(), UNKNOWN_SHARDNUM,
-              UNKNOWN_NUMSHARDS);
-        } else {
-          writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, UNKNOWN_NUMSHARDS);
+    Writer<T> getWriter(BoundedWindow window, PaneInfo paneInfo) throws Exception {
+      Writer<T> writer;
+      if (windowedWrites) {
+        // If we are doing windowed writes, we need to ensure that we have separate files for
+        // data in different windows.
+        KV<BoundedWindow, PaneInfo> key = KV.of(window, paneInfo);
+        writer = windowedWriters.get(key);
+        if (writer == null) {
+          LOG.info("Opening writer for write operation {}, window {}", writeOperation, window);
+          writer = writeOperation.createWriter();
+          writer.openWindowed(UUID.randomUUID().toString(), window, paneInfo, UNKNOWN_SHARDNUM);
+          windowedWriters.put(key, writer);
+          LOG.debug("Done opening writer {} for operation {} window {}", writer, writeOperation,
+              window);
+        }
+      } else {
+        // Unwindowed writes. Just cache a single writer for the bundle.
+        writer = this.writer;
+        if (writer == null) {
+          LOG.info("Opening writer for write operation {}", writeOperation);
+          writer = this.writer = writeOperation.createWriter();
+          writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
+          LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
         }
         this.window = window;
         LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
       }
+      return writer;
+    }
+
+    @StartBundle
+    public void startBundle(StartBundleContext c) {
+      // Reset state in case of reuse. We need to make sure that each bundle gets unique writers.
+      writer = null;
+      if (windowedWrites) {
+        windowedWriters = Maps.newHashMap();
+      }
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
+      Writer<T> writer = getWriter(window, c.pane());
       try {
         writer.write(c.element());
       } catch (Exception e) {
@@ -292,6 +327,13 @@ public void finishBundle(FinishBundleContext c) throws Exception {
         c.output(result, window.maxTimestamp(), window);
         // Reset state in case of reuse.
         writer = null;
+      } else {
+        for (Map.Entry<KV<BoundedWindow, PaneInfo>, Writer<T>> entry :
+            windowedWriters.entrySet()) {
+          FileResult result = entry.getValue().close();
+          BoundedWindow window = entry.getKey().getKey();
+          c.output(result, window.maxTimestamp(), window);
+        }
       }
     }
 
@@ -308,24 +350,16 @@ public void populateDisplayData(DisplayData.Builder builder) {
    * @see WriteBundles
    */
   private class WriteShardedBundles extends DoFn<KV<Integer, Iterable<T>>, FileResult> {
-    private final PCollectionView<Integer> numShardsView;
-
-    WriteShardedBundles(PCollectionView<Integer> numShardsView) {
-      this.numShardsView = numShardsView;
-    }
-
     @ProcessElement
     public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
-      int numShards = numShardsView != null ? c.sideInput(numShardsView) : getNumShards().get();
       // In a sharded write, single input element represents one shard. We can open and close
       // the writer in each call to processElement.
       LOG.info("Opening writer for write operation {}", writeOperation);
       Writer<T> writer = writeOperation.createWriter();
       if (windowedWrites) {
-        writer.openWindowed(UUID.randomUUID().toString(), window, c.pane(), c.element().getKey(),
-            numShards);
+        writer.openWindowed(UUID.randomUUID().toString(), window, c.pane(), c.element().getKey());
       } else {
-        writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, UNKNOWN_NUMSHARDS);
+        writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
       }
       LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
 
@@ -378,7 +412,7 @@ private static class ApplyShardingKey<T> extends DoFn<T, KV<Integer, T>> {
 
     @ProcessElement
     public void processElement(ProcessContext context) {
-      int shardCount = 0;
+      final int shardCount;
       if (numShardsView != null) {
         shardCount = context.sideInput(numShardsView);
       } else {
@@ -448,35 +482,33 @@ private PDone createWrite(PCollection<T> input) {
     // initial ParDo.
     PCollection<FileResult> results;
     final PCollectionView<Integer> numShardsView;
+    Coder<BoundedWindow> shardedWindowCoder =
+        (Coder<BoundedWindow>) input.getWindowingStrategy().getWindowFn().windowCoder();
     if (computeNumShards == null && numShardsProvider == null) {
-      if (windowedWrites) {
-        throw new IllegalStateException("When doing windowed writes, numShards must be set"
-            + "explicitly to a positive value");
-      }
       numShardsView = null;
-      results = input
-          .apply("WriteBundles",
-              ParDo.of(new WriteBundles()));
+      results = input.apply("WriteBundles", ParDo.of(new WriteBundles()));
     } else {
+      List<PCollectionView<?>> sideInputs = Lists.newArrayList();
       if (computeNumShards != null) {
         numShardsView = input.apply(computeNumShards);
-        results  = input
-            .apply("ApplyShardLabel", ParDo.of(
-                new ApplyShardingKey<T>(numShardsView, null)).withSideInputs(numShardsView))
-            .apply("GroupIntoShards", GroupByKey.<Integer, T>create())
-            .apply("WriteShardedBundles",
-                ParDo.of(new WriteShardedBundles(numShardsView))
-                    .withSideInputs(numShardsView));
+        sideInputs.add(numShardsView);
       } else {
         numShardsView = null;
-        results = input
-            .apply("ApplyShardLabel", ParDo.of(new ApplyShardingKey<T>(null, numShardsProvider)))
-            .apply("GroupIntoShards", GroupByKey.<Integer, T>create())
-            .apply("WriteShardedBundles",
-                ParDo.of(new WriteShardedBundles(null)));
       }
+
+      PCollection<KV<Integer, Iterable<T>>> sharded =
+          input
+              .apply("ApplyShardLabel", ParDo.of(
+                  new ApplyShardingKey<T>(numShardsView,
+                      (numShardsView != null) ? null : numShardsProvider))
+                  .withSideInputs(sideInputs))
+              .apply("GroupIntoShards", GroupByKey.<Integer, T>create());
+      shardedWindowCoder =
+          (Coder<BoundedWindow>) sharded.getWindowingStrategy().getWindowFn().windowCoder();
+
+      results = sharded.apply("WriteShardedBundles", ParDo.of(new WriteShardedBundles()));
     }
-    results.setCoder(FileResultCoder.of());
+    results.setCoder(FileResultCoder.of(shardedWindowCoder));
 
     if (windowedWrites) {
       // When processing streaming windowed writes, results will arrive multiple times. This
@@ -486,7 +518,8 @@ private PDone createWrite(PCollection<T> input) {
       // whenever new data arrives.
       PCollection<KV<Void, FileResult>> keyedResults =
           results.apply("AttachSingletonKey", WithKeys.<Void, FileResult>of((Void) null));
-      keyedResults.setCoder(KvCoder.of(VoidCoder.of(), FileResultCoder.of()));
+      keyedResults.setCoder(KvCoder.of(VoidCoder.of(),
+          FileResultCoder.of(shardedWindowCoder)));
 
       // Is the continuation trigger sufficient?
       keyedResults
@@ -497,7 +530,7 @@ public void processElement(ProcessContext c) throws Exception {
               LOG.info("Finalizing write operation {}.", writeOperation);
               List<FileResult> results = Lists.newArrayList(c.element().getValue());
               writeOperation.finalize(results);
-              LOG.debug("Done finalizing write operation {}", writeOperation);
+              LOG.debug("Done finalizing write operation");
             }
           }));
     } else {
@@ -543,8 +576,7 @@ public void processElement(ProcessContext c) throws Exception {
                     extraShardsNeeded, results.size(), minShardsNeeded);
                 for (int i = 0; i < extraShardsNeeded; ++i) {
                   Writer<T> writer = writeOperation.createWriter();
-                  writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM,
-                      UNKNOWN_NUMSHARDS);
+                  writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
                   FileResult emptyWrite = writer.close();
                   results.add(emptyWrite);
                 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
index c3f2a589b739..caad75989cbd 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
@@ -67,8 +67,7 @@
  */
 @RunWith(JUnit4.class)
 public class FileBasedSinkTest {
-  @Rule
-  public TemporaryFolder tmpFolder = new TemporaryFolder();
+  @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
 
   private final String tempDirectoryName = "temp";
 
@@ -88,13 +87,13 @@ private ResourceId getBaseTempDirectory() {
   }
 
   /**
-   * Writer opens the correct file, writes the header, footer, and elements in the
-   * correct order, and returns the correct filename.
+   * Writer opens the correct file, writes the header, footer, and elements in the correct
+   * order, and returns the correct filename.
    */
   @Test
   public void testWriter() throws Exception {
     String testUid = "testId";
-    ResourceId expectedFile = getBaseTempDirectory()
+    ResourceId expectedTempFile = getBaseTempDirectory()
         .resolve(testUid, StandardResolveOptions.RESOLVE_FILE);
     List<String> values = Arrays.asList("sympathetic vulture", "boresome hummingbird");
     List<String> expected = new ArrayList<>();
@@ -104,14 +103,15 @@ public void testWriter() throws Exception {
 
     SimpleSink.SimpleWriter writer =
         buildWriteOperationWithTempDir(getBaseTempDirectory()).createWriter();
-    writer.openUnwindowed(testUid, -1, -1);
+    writer.openUnwindowed(testUid, -1);
     for (String value : values) {
       writer.write(value);
     }
     FileResult result = writer.close();
 
-    assertEquals(expectedFile, result.getFilename());
-    assertFileContains(expected, expectedFile);
+    FileBasedSink sink = writer.getWriteOperation().getSink();
+    assertEquals(expectedTempFile, result.getTempFilename());
+    assertFileContains(expected, expectedTempFile);
   }
 
   /**
@@ -131,9 +131,7 @@ private void assertFileContains(List<String> expected, ResourceId file) throws E
     }
   }
 
-  /**
-   * Write lines to a file.
-   */
+  /** Write lines to a file. */
   private void writeFile(List<String> lines, File file) throws Exception {
     try (PrintWriter writer = new PrintWriter(new FileOutputStream(file))) {
       for (String line : lines) {
@@ -150,18 +148,14 @@ public void testRemoveWithTempFilename() throws Exception {
     testRemoveTemporaryFiles(3, getBaseTempDirectory());
   }
 
-  /**
-   * Finalize copies temporary files to output files and removes any temporary files.
-   */
+  /** Finalize copies temporary files to output files and removes any temporary files. */
   @Test
   public void testFinalize() throws Exception {
     List<File> files = generateTemporaryFilesForFinalize(3);
     runFinalize(buildWriteOperation(), files);
   }
 
-  /**
-   * Finalize can be called repeatedly.
-   */
+  /** Finalize can be called repeatedly. */
   @Test
   public void testFinalizeMultipleCalls() throws Exception {
     List<File> files = generateTemporaryFilesForFinalize(3);
@@ -170,9 +164,7 @@ public void testFinalizeMultipleCalls() throws Exception {
     runFinalize(writeOp, files);
   }
 
-  /**
-   * Finalize can be called when some temporary files do not exist and output files exist.
-   */
+  /** Finalize can be called when some temporary files do not exist and output files exist. */
   @Test
   public void testFinalizeWithIntermediateState() throws Exception {
     SimpleSink.SimpleWriteOperation writeOp = buildWriteOperation();
@@ -186,9 +178,7 @@ public void testFinalizeWithIntermediateState() throws Exception {
     runFinalize(writeOp, files);
   }
 
-  /**
-   * Generate n temporary files using the temporary file pattern of Writer.
-   */
+  /** Generate n temporary files using the temporary file pattern of Writer. */
   private List<File> generateTemporaryFilesForFinalize(int numFiles) throws Exception {
     List<File> temporaryFiles = new ArrayList<>();
     for (int i = 0; i < numFiles; i++) {
@@ -203,18 +193,20 @@ private List<File> generateTemporaryFilesForFinalize(int numFiles) throws Except
     return temporaryFiles;
   }
 
-  /**
-   * Finalize and verify that files are copied and temporary files are optionally removed.
-   */
+  /** Finalize and verify that files are copied and temporary files are optionally removed. */
   private void runFinalize(SimpleSink.SimpleWriteOperation writeOp, List<File> temporaryFiles)
       throws Exception {
     int numFiles = temporaryFiles.size();
 
     List<FileResult> fileResults = new ArrayList<>();
     // Create temporary output bundles and output File objects.
-    for (File f : temporaryFiles) {
-      ResourceId file = LocalResources.fromFile(f, false);
-      fileResults.add(new FileResult(file, null));
+    for (int i = 0; i < numFiles; i++) {
+      fileResults.add(
+          new FileResult(
+              LocalResources.fromFile(temporaryFiles.get(i), false),
+              WriteFiles.UNKNOWN_SHARDNUM,
+              null,
+              null));
     }
 
     writeOp.finalize(fileResults);
@@ -233,8 +225,8 @@ private void runFinalize(SimpleSink.SimpleWriteOperation writeOp, List<File> tem
   }
 
   /**
-   * Create n temporary and output files and verify that removeTemporaryFiles only
-   * removes temporary files.
+   * Create n temporary and output files and verify that removeTemporaryFiles only removes temporary
+   * files.
    */
   private void testRemoveTemporaryFiles(int numFiles, ResourceId tempDirectory)
       throws Exception {
@@ -276,9 +268,7 @@ private void testRemoveTemporaryFiles(int numFiles, ResourceId tempDirectory)
     }
   }
 
-  /**
-   * Output files are copied to the destination location with the correct names and contents.
-   */
+  /** Output files are copied to the destination location with the correct names and contents. */
   @Test
   public void testCopyToOutputFiles() throws Exception {
     SimpleSink.SimpleWriteOperation writeOp = buildWriteOperation();
@@ -329,6 +319,7 @@ public List<ResourceId> generateDestinationFilenames(
   /**
    * Output filenames are generated correctly when an extension is supplied.
    */
+
   @Test
   public void testGenerateOutputFilenames() {
     List<ResourceId> expected;
@@ -357,9 +348,7 @@ public void testGenerateOutputFilenames() {
     assertEquals(expected, actual);
   }
 
-  /**
-   * Reject non-distinct output filenames.
-   */
+  /** Reject non-distinct output filenames. */
   @Test
   public void testCollidingOutputFilenames() throws IOException {
     ResourceId root = getBaseOutputDirectory();
@@ -372,22 +361,19 @@ public void testCollidingOutputFilenames() throws IOException {
     ResourceId output = root.resolve("file-03.test", StandardResolveOptions.RESOLVE_FILE);
     // More than one shard does.
     try {
-      Iterable<FileResult> results = Lists.newArrayList(
-          new FileResult(temp1, output),
-          new FileResult(temp2, output),
-          new FileResult(temp3, output));
-
+      Iterable<FileResult> results =
+          Lists.newArrayList(
+              new FileResult(temp1, 1, null, null),
+              new FileResult(temp2, 1, null, null),
+              new FileResult(temp3, 1, null, null));
       writeOp.buildOutputFilenames(results);
       fail("Should have failed.");
     } catch (IllegalStateException exn) {
-      assertEquals("Only generated 1 distinct file names for 3 files.",
-                   exn.getMessage());
+      assertEquals("Only generated 1 distinct file names for 3 files.", exn.getMessage());
     }
   }
 
-  /**
-   * Output filenames are generated correctly when an extension is not supplied.
-   */
+  /** Output filenames are generated correctly when an extension is not supplied. */
   @Test
   public void testGenerateOutputFilenamesWithoutExtension() {
     List<ResourceId> expected;
@@ -415,53 +401,59 @@ public void testGenerateOutputFilenamesWithoutExtension() {
     assertEquals(expected, actual);
   }
 
-  /**
-   * {@link CompressionType#BZIP2} correctly writes BZip2 data.
-   */
+  /** {@link CompressionType#BZIP2} correctly writes BZip2 data. */
   @Test
   public void testCompressionTypeBZIP2() throws FileNotFoundException, IOException {
     final File file =
         writeValuesWithWritableByteChannelFactory(CompressionType.BZIP2, "abc", "123");
     // Read Bzip2ed data back in using Apache commons API (de facto standard).
-    assertReadValues(new BufferedReader(new InputStreamReader(
-        new BZip2CompressorInputStream(new FileInputStream(file)), StandardCharsets.UTF_8.name())),
-        "abc", "123");
+    assertReadValues(
+        new BufferedReader(
+            new InputStreamReader(
+                new BZip2CompressorInputStream(new FileInputStream(file)),
+                StandardCharsets.UTF_8.name())),
+        "abc",
+        "123");
   }
 
-  /**
-   * {@link CompressionType#GZIP} correctly writes Gzipped data.
-   */
+  /** {@link CompressionType#GZIP} correctly writes Gzipped data. */
   @Test
   public void testCompressionTypeGZIP() throws FileNotFoundException, IOException {
     final File file = writeValuesWithWritableByteChannelFactory(CompressionType.GZIP, "abc", "123");
     // Read Gzipped data back in using standard API.
-    assertReadValues(new BufferedReader(new InputStreamReader(
-        new GZIPInputStream(new FileInputStream(file)), StandardCharsets.UTF_8.name())), "abc",
+    assertReadValues(
+        new BufferedReader(
+            new InputStreamReader(
+                new GZIPInputStream(new FileInputStream(file)), StandardCharsets.UTF_8.name())),
+        "abc",
         "123");
   }
 
-  /**
-   * {@link CompressionType#DEFLATE} correctly writes deflate data.
-   */
+  /** {@link CompressionType#DEFLATE} correctly writes deflate data. */
   @Test
   public void testCompressionTypeDEFLATE() throws FileNotFoundException, IOException {
-    final File file = writeValuesWithWritableByteChannelFactory(
-        CompressionType.DEFLATE, "abc", "123");
+    final File file =
+        writeValuesWithWritableByteChannelFactory(CompressionType.DEFLATE, "abc", "123");
     // Read Gzipped data back in using standard API.
-    assertReadValues(new BufferedReader(new InputStreamReader(new DeflateCompressorInputStream(
-        new FileInputStream(file)), StandardCharsets.UTF_8.name())), "abc", "123");
+    assertReadValues(
+        new BufferedReader(
+            new InputStreamReader(
+                new DeflateCompressorInputStream(new FileInputStream(file)),
+                StandardCharsets.UTF_8.name())),
+        "abc",
+        "123");
   }
 
-  /**
-   * {@link CompressionType#UNCOMPRESSED} correctly writes uncompressed data.
-   */
+  /** {@link CompressionType#UNCOMPRESSED} correctly writes uncompressed data. */
   @Test
   public void testCompressionTypeUNCOMPRESSED() throws FileNotFoundException, IOException {
     final File file =
         writeValuesWithWritableByteChannelFactory(CompressionType.UNCOMPRESSED, "abc", "123");
     // Read uncompressed data back in using standard API.
-    assertReadValues(new BufferedReader(new InputStreamReader(
-        new FileInputStream(file), StandardCharsets.UTF_8.name())), "abc",
+    assertReadValues(
+        new BufferedReader(
+            new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8.name())),
+        "abc",
         "123");
   }
 
@@ -487,8 +479,8 @@ private File writeValuesWithWritableByteChannelFactory(final WritableByteChannel
   }
 
   /**
-   * {@link Writer} writes to the {@link WritableByteChannel} provided by
-   * {@link DrunkWritableByteChannelFactory}.
+   * {@link Writer} writes to the {@link WritableByteChannel} provided by {@link
+   * DrunkWritableByteChannelFactory}.
    */
   @Test
   public void testFileBasedWriterWithWritableByteChannelFactory() throws Exception {
@@ -511,18 +503,16 @@ public void testFileBasedWriterWithWritableByteChannelFactory() throws Exception
     expected.add("footer");
     expected.add("footer");
 
-    writer.openUnwindowed(testUid, -1, -1);
+    writer.openUnwindowed(testUid, -1);
     writer.write("a");
     writer.write("b");
     final FileResult result = writer.close();
 
-    assertEquals(expectedFile, result.getFilename());
+    assertEquals(expectedFile, result.getTempFilename());
     assertFileContains(expected, expectedFile);
   }
 
-  /**
-   * Build a SimpleSink with default options.
-   */
+  /** Build a SimpleSink with default options. */
   private SimpleSink buildSink() {
     return new SimpleSink(getBaseOutputDirectory(), "file", "-SS-of-NN", ".test");
   }
@@ -535,9 +525,7 @@ private SimpleSink.SimpleWriteOperation buildWriteOperationWithTempDir(ResourceI
     return new SimpleSink.SimpleWriteOperation(sink, tempDirectory);
   }
 
-  /**
-   * Build a write operation with the default options for it and its parent sink.
-   */
+  /** Build a write operation with the default options for it and its parent sink. */
   private SimpleSink.SimpleWriteOperation buildWriteOperation() {
     return buildSink().createWriteOperation();
   }

From a09a3ec0645689f54a4fffc944cc531192e1f014 Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Tue, 9 May 2017 15:10:07 -0700
Subject: [PATCH 177/387] Simpler code for setting shard numbers on results in
 FileBasedSink

---
 .../org/apache/beam/sdk/io/FileBasedSink.java | 113 +++++++++++-------
 1 file changed, 68 insertions(+), 45 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 21177941d8f5..7f729a7ea828 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -22,10 +22,12 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Verify.verifyNotNull;
+import static org.apache.beam.sdk.io.WriteFiles.UNKNOWN_SHARDNUM;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
 import java.io.IOException;
 import java.io.InputStream;
@@ -530,44 +532,63 @@ protected final Map<ResourceId, ResourceId> buildOutputFilenames(
       int numShards = Iterables.size(writerResults);
       Map<ResourceId, ResourceId> outputFilenames = new HashMap<>();
 
-      List<FileResult> unshardedFiles = new ArrayList<>();
       FilenamePolicy policy = getSink().getFilenamePolicy();
+      ResourceId baseOutputDir = getSink().getBaseOutputDirectoryProvider().get();
+
+      // Either all results have a shard number set (if the sink is configured with a fixed
+      // number of shards), or they all don't (otherwise).
+      Boolean isShardNumberSetEverywhere = null;
       for (FileResult result : writerResults) {
-        if (result.getShard() != WriteFiles.UNKNOWN_SHARDNUM) {
-          outputFilenames.put(result.getTempFilename(),
-              result.getDestinationFile(policy, getSink().getBaseOutputDirectoryProvider().get(),
-                  numShards, getSink().getExtension()));
+        boolean isShardNumberSetHere = (result.getShard() != UNKNOWN_SHARDNUM);
+        if (isShardNumberSetEverywhere == null) {
+          isShardNumberSetEverywhere = isShardNumberSetHere;
         } else {
-          unshardedFiles.add(result);
+          checkArgument(
+              isShardNumberSetEverywhere == isShardNumberSetHere,
+              "Found a mix of files with and without shard number set: %s",
+              result);
         }
       }
 
-      // writerResults won't contain destination filenames, so we dynamically generate them here.
-      if (unshardedFiles.size() > 0) {
-        checkArgument(outputFilenames.isEmpty());
+      if (isShardNumberSetEverywhere == null) {
+        isShardNumberSetEverywhere = true;
+      }
 
+      List<FileResult> resultsWithShardNumbers = Lists.newArrayList();
+      if (isShardNumberSetEverywhere) {
+        resultsWithShardNumbers = Lists.newArrayList(writerResults);
+      } else {
         // Sort files for idempotence. Sort by temporary filename.
         // Note that this codepath should not be used when processing triggered windows. In the
         // case of triggers, the list of FileResult objects in the Finalize iterable is not
         // deterministic, and might change over retries. This breaks the assumption below that
         // sorting the FileResult objects provides idempotency.
-        unshardedFiles = Ordering.from(new Comparator<FileResult>() {
-          @Override
-          public int compare(FileResult first, FileResult second) {
-            return first.getTempFilename().toString().compareTo(
-                second.getTempFilename().toString());
-          }
-        }).sortedCopy(unshardedFiles);
-        for (int i = 0; i < unshardedFiles.size(); i++) {
-          FileResult result = unshardedFiles.get(i);
-          result.setShard(i);
-          outputFilenames.put(result.getTempFilename(),
-              result.getDestinationFile(policy, getSink().getBaseOutputDirectoryProvider().get(),
-                  numShards, getSink().getExtension()));
+        List<FileResult> sortedByTempFilename =
+            Ordering.from(
+                new Comparator<FileResult>() {
+                  @Override
+                  public int compare(FileResult first, FileResult second) {
+                    String firstFilename = first.getTempFilename().toString();
+                    String secondFilename = second.getTempFilename().toString();
+                    return firstFilename.compareTo(secondFilename);
+                  }
+                })
+                .sortedCopy(writerResults);
+        for (int i = 0; i < sortedByTempFilename.size(); i++) {
+          resultsWithShardNumbers.add(sortedByTempFilename.get(i).withShard(i));
         }
       }
 
-      int numDistinctShards = new HashSet<ResourceId>(outputFilenames.values()).size();
+      for (FileResult result : resultsWithShardNumbers) {
+        checkArgument(
+            result.getShard() != UNKNOWN_SHARDNUM, "Should have set shard number on %s", result);
+        outputFilenames.put(
+            result.getTempFilename(),
+            result.getDestinationFile(
+                policy, baseOutputDir, numShards, getSink().getExtension()));
+      }
+
+      int numDistinctShards = new HashSet<>(outputFilenames.values()).size();
       checkState(numDistinctShards == outputFilenames.size(),
          "Only generated %s distinct file names for %s files.",
          numDistinctShards, outputFilenames.size());
@@ -920,12 +941,11 @@ public WriteOperation<T> getWriteOperation() {
    */
   public static final class FileResult {
     private final ResourceId tempFilename;
-    private int shard;
-    private BoundedWindow window;
-    private PaneInfo paneInfo;
+    private final int shard;
+    private final BoundedWindow window;
+    private final PaneInfo paneInfo;
 
-    public FileResult(ResourceId tempFilename, int shard, BoundedWindow window,
-                      PaneInfo paneInfo) {
+    public FileResult(ResourceId tempFilename, int shard, BoundedWindow window, PaneInfo paneInfo) {
       this.tempFilename = tempFilename;
       this.shard = shard;
       this.window = window;
@@ -940,8 +960,8 @@ public int getShard() {
       return shard;
     }
 
-    public void setShard(int shard) {
-      this.shard = shard;
+    public FileResult withShard(int shard) {
+      return new FileResult(tempFilename, shard, window, paneInfo);
     }
 
     public BoundedWindow getWindow() {
@@ -954,7 +974,7 @@ public PaneInfo getPaneInfo() {
 
     public ResourceId getDestinationFile(FilenamePolicy policy, ResourceId outputDirectory,
                                          int numShards, String extension) {
-      checkArgument(getShard() != WriteFiles.UNKNOWN_SHARDNUM);
+      checkArgument(getShard() != UNKNOWN_SHARDNUM);
       checkArgument(numShards > 0);
       if (getWindow() != null) {
         return policy.windowedFilename(outputDirectory, new WindowedContext(
@@ -966,8 +986,11 @@ public ResourceId getDestinationFile(FilenamePolicy policy, ResourceId outputDir
     }
 
     public String toString() {
-      return MoreObjects.toStringHelper(FileBasedSink.FileResult.class)
+      return MoreObjects.toStringHelper(FileResult.class)
           .add("tempFilename", tempFilename)
+          .add("shard", shard)
+          .add("window", window)
+          .add("paneInfo", paneInfo)
           .toString();
     }
   }
@@ -976,9 +999,10 @@ public String toString() {
    * A coder for {@link FileResult} objects.
    */
   public static final class FileResultCoder extends StructuredCoder<FileResult> {
-    private final Coder<String> stringCoder = StringUtf8Coder.of();
-    private final Coder<Integer> integerCoder = VarIntCoder.of();
-    private final Coder<PaneInfo> paneInfoCoder = NullableCoder.of(PaneInfoCoder.INSTANCE);
+    private static final Coder<String> FILENAME_CODER = StringUtf8Coder.of();
+    private static final Coder<Integer> SHARD_CODER = VarIntCoder.of();
+    private static final Coder<PaneInfo> PANE_INFO_CODER = NullableCoder.of(PaneInfoCoder.INSTANCE);
+
     private final Coder<BoundedWindow> windowCoder;
 
     protected FileResultCoder(Coder<BoundedWindow> windowCoder) {
@@ -1000,30 +1024,29 @@ public void encode(FileResult value, OutputStream outStream)
       if (value == null) {
         throw new CoderException("cannot encode a null value");
       }
-      stringCoder.encode(value.getTempFilename().toString(), outStream);
+      FILENAME_CODER.encode(value.getTempFilename().toString(), outStream);
       windowCoder.encode(value.getWindow(), outStream);
-      paneInfoCoder.encode(value.getPaneInfo(), outStream);
-      integerCoder.encode(value.getShard(), outStream);
+      PANE_INFO_CODER.encode(value.getPaneInfo(), outStream);
+      SHARD_CODER.encode(value.getShard(), outStream);
     }
 
     @Override
     public FileResult decode(InputStream inStream)
         throws IOException {
-      String tempFilename = stringCoder.decode(inStream);
-      assert tempFilename != null;  // fixes a compiler warning
+      String tempFilename = FILENAME_CODER.decode(inStream);
       BoundedWindow window = windowCoder.decode(inStream);
-      PaneInfo paneInfo = paneInfoCoder.decode(inStream);
-      int shard = integerCoder.decode(inStream);
+      PaneInfo paneInfo = PANE_INFO_CODER.decode(inStream);
+      int shard = SHARD_CODER.decode(inStream);
       return new FileResult(FileSystems.matchNewResource(tempFilename, false /* isDirectory */),
           shard, window, paneInfo);
     }
 
     @Override
     public void verifyDeterministic() throws NonDeterministicException {
-      stringCoder.verifyDeterministic();
+      FILENAME_CODER.verifyDeterministic();
       windowCoder.verifyDeterministic();
-      paneInfoCoder.verifyDeterministic();
-      integerCoder.verifyDeterministic();
+      PANE_INFO_CODER.verifyDeterministic();
+      SHARD_CODER.verifyDeterministic();
     }
   }
 

From bb6df582ac4b5a0af724fdf6dfb6cd5660f6392b Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Tue, 9 May 2017 15:25:57 -0700
Subject: [PATCH 178/387] Splits WriteBundles into windowed/unwindowed versions

---
 .../org/apache/beam/sdk/io/WriteFiles.java    | 179 +++++++++---------
 1 file changed, 93 insertions(+), 86 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
index cab58621be38..0786e5d72fe9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
@@ -250,91 +250,93 @@ public WriteFiles<T> withWindowedWrites() {
 
   /**
    * Writes all the elements in a bundle using a {@link Writer} produced by the
-   * {@link WriteOperation} associated with the {@link FileBasedSink}.
+   * {@link WriteOperation} associated with the {@link FileBasedSink} with windowed writes enabled.
    */
-  private class WriteBundles extends DoFn<T, FileResult> {
-    // Writer that will write the records in this bundle. Lazily
-    // initialized in processElement.
-    private Writer<T> writer = null;
-    private BoundedWindow window = null;
+  private class WriteWindowedBundles extends DoFn<T, FileResult> {
     private Map<KV<BoundedWindow, PaneInfo>, Writer<T>> windowedWriters;
 
-    Writer<T> getWriter(BoundedWindow window, PaneInfo paneInfo) throws Exception {
+    @StartBundle
+    public void startBundle(StartBundleContext c) {
+      // Reset state in case of reuse. We need to make sure that each bundle gets unique writers.
+      windowedWriters = Maps.newHashMap();
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
+      PaneInfo paneInfo = c.pane();
       Writer<T> writer;
-      if (windowedWrites) {
-        // If we are doing windowed writes, we need to ensure that we have separate files for
-        // data in different windows.
-        KV<BoundedWindow, PaneInfo> key = KV.of(window, paneInfo);
-        writer = windowedWriters.get(key);
-        if (writer == null) {
-          LOG.info("Opening writer for write operation {}, window {}", writeOperation, window);
-          writer = writeOperation.createWriter();
-          writer.openWindowed(UUID.randomUUID().toString(), window, paneInfo, UNKNOWN_SHARDNUM);
-          windowedWriters.put(key, writer);
-          LOG.debug("Done opening writer {} for operation {} window {}", writer, writeOperation,
-              window);
-        }
-      } else {
-        // Unwindowed writes. Just cache a single writer for the bundle.
-        writer = this.writer;
-        if (writer == null) {
-          LOG.info("Opening writer for write operation {}", writeOperation);
-          writer = this.writer = writeOperation.createWriter();
-          writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
-          LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
-        }
-        this.window = window;
-        LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
+      // If we are doing windowed writes, we need to ensure that we have separate files for
+      // data in different windows/panes.
+      KV<BoundedWindow, PaneInfo> key = KV.of(window, paneInfo);
+      writer = windowedWriters.get(key);
+      if (writer == null) {
+        String uuid = UUID.randomUUID().toString();
+        LOG.info(
+            "Opening writer {} for write operation {}, window {} pane {}",
+            uuid,
+            writeOperation,
+            window,
+            paneInfo);
+        writer = writeOperation.createWriter();
+        writer.openWindowed(uuid, window, paneInfo, UNKNOWN_SHARDNUM);
+        windowedWriters.put(key, writer);
+        LOG.debug("Done opening writer");
+      }
+
+      writeOrClose(writer, c.element());
+    }
+
+    @FinishBundle
+    public void finishBundle(FinishBundleContext c) throws Exception {
+      for (Map.Entry<KV<BoundedWindow, PaneInfo>, Writer<T>> entry : windowedWriters.entrySet()) {
+        FileResult result = entry.getValue().close();
+        BoundedWindow window = entry.getKey().getKey();
+        c.output(result, window.maxTimestamp(), window);
       }
-      return writer;
     }
 
+    @Override
+    public void populateDisplayData(DisplayData.Builder builder) {
+      builder.delegate(WriteFiles.this);
+    }
+  }
+
+  /**
+   * Writes all the elements in a bundle using a {@link Writer} produced by the
+   * {@link WriteOperation} associated with the {@link FileBasedSink} with windowed writes disabled.
+   */
+  private class WriteUnwindowedBundles extends DoFn<T, FileResult> {
+    // Writer that will write the records in this bundle. Lazily
+    // initialized in processElement.
+    private Writer<T> writer = null;
+    private BoundedWindow window = null;
+
     @StartBundle
     public void startBundle(StartBundleContext c) {
       // Reset state in case of reuse. We need to make sure that each bundle gets unique writers.
       writer = null;
-      if (windowedWrites) {
-        windowedWriters = Maps.newHashMap();
-      }
     }
 
     @ProcessElement
     public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
-      Writer<T> writer = getWriter(window, c.pane());
-      try {
-        writer.write(c.element());
-      } catch (Exception e) {
-        // Discard write result and close the write.
-        try {
-          writer.close();
-          // The writer does not need to be reset, as this DoFn cannot be reused.
-        } catch (Exception closeException) {
-          if (closeException instanceof InterruptedException) {
-            // Do not silently ignore interrupted state.
-            Thread.currentThread().interrupt();
-          }
-          // Do not mask the exception that caused the write to fail.
-          e.addSuppressed(closeException);
-        }
-        throw e;
+      // Cache a single writer for the bundle.
+      if (writer == null) {
+        LOG.info("Opening writer for write operation {}", writeOperation);
+        writer = writeOperation.createWriter();
+        writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
+        LOG.debug("Done opening writer");
       }
+      this.window = window;
+      writeOrClose(this.writer, c.element());
     }
 
     @FinishBundle
     public void finishBundle(FinishBundleContext c) throws Exception {
-      if (writer != null) {
-        FileResult result = writer.close();
-        c.output(result, window.maxTimestamp(), window);
-        // Reset state in case of reuse.
-        writer = null;
-      } else {
-        for (Map.Entry<KV<BoundedWindow, PaneInfo>, Writer<T>> entry :
-            windowedWriters.entrySet()) {
-          FileResult result = entry.getValue().close();
-          BoundedWindow window = entry.getKey().getKey();
-          c.output(result, window.maxTimestamp(), window);
-        }
+      if (writer == null) {
+        return;
       }
+      FileResult result = writer.close();
+      c.output(result, window.maxTimestamp(), window);
     }
 
     @Override
@@ -344,10 +346,8 @@ public void populateDisplayData(DisplayData.Builder builder) {
   }
 
   /**
-   * Like {@link WriteBundles}, but where the elements for each shard have been collected into
-   * a single iterable.
-   *
-   * @see WriteBundles
+   * Like {@link WriteWindowedBundles} and {@link WriteUnwindowedBundles}, but where the elements
+   * for each shard have been collected into a single iterable.
    */
   private class WriteShardedBundles extends DoFn<KV<Integer, Iterable<T>>, FileResult> {
     @ProcessElement
@@ -361,25 +361,11 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
       } else {
         writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM);
       }
-      LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
+      LOG.debug("Done opening writer");
 
       try {
-        try {
-          for (T t : c.element().getValue()) {
-            writer.write(t);
-          }
-        } catch (Exception e) {
-          try {
-            writer.close();
-          } catch (Exception closeException) {
-            if (closeException instanceof InterruptedException) {
-              // Do not silently ignore interrupted state.
-              Thread.currentThread().interrupt();
-            }
-            // Do not mask the exception that caused the write to fail.
-            e.addSuppressed(closeException);
-          }
-          throw e;
+        for (T t : c.element().getValue()) {
+          writeOrClose(writer, t);
         }
 
         // Close the writer; if this throws let the error propagate.
@@ -398,6 +384,24 @@ public void populateDisplayData(DisplayData.Builder builder) {
     }
   }
 
+  private static <T> void writeOrClose(Writer<T> writer, T t) throws Exception {
+    try {
+      writer.write(t);
+    } catch (Exception e) {
+      try {
+        writer.close();
+      } catch (Exception closeException) {
+        if (closeException instanceof InterruptedException) {
+          // Do not silently ignore interrupted state.
+          Thread.currentThread().interrupt();
+        }
+        // Do not mask the exception that caused the write to fail.
+        e.addSuppressed(closeException);
+      }
+      throw e;
+    }
+  }
+
   private static class ApplyShardingKey<T> extends DoFn<T, KV<Integer, T>> {
     private final PCollectionView<Integer> numShardsView;
     private final ValueProvider<Integer> numShardsProvider;
@@ -486,7 +490,10 @@ private PDone createWrite(PCollection<T> input) {
         (Coder<BoundedWindow>) input.getWindowingStrategy().getWindowFn().windowCoder();
     if (computeNumShards == null && numShardsProvider == null) {
       numShardsView = null;
-      results = input.apply("WriteBundles", ParDo.of(new WriteBundles()));
+      results =
+          input.apply(
+              "WriteBundles",
+              ParDo.of(windowedWrites ? new WriteWindowedBundles() : new WriteUnwindowedBundles()));
     } else {
       List<PCollectionView<?>> sideInputs = Lists.newArrayList();
       if (computeNumShards != null) {

From 1badcb7d15fda696e77e3b2221c6812773715b05 Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Wed, 10 May 2017 12:27:55 -0700
Subject: [PATCH 179/387] Adds dependency on findbugs to examples/java

---
 examples/java/pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 21096f74bf16..6f0776cc1a71 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -472,6 +472,11 @@
       <artifactId>google-api-services-bigquery</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.google.http-client</groupId>
       <artifactId>google-http-client</artifactId>

From baa178359ced13ef50f8f8ba398567836bbb63a0 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Tue, 9 May 2017 15:19:34 -0700
Subject: [PATCH 180/387] [BEAM-2240] Always augment exception with step name.

---
 sdks/python/apache_beam/pipeline_test.py      |  2 +-
 sdks/python/apache_beam/runners/common.py     | 24 +++++++++++++------
 .../maptask_executor_runner_test.py           | 19 +++++++++++++--
 3 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py
index c6b1e48fb5b5..8aa8a8aa0292 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -278,7 +278,7 @@ def raise_exception(exn):
     with self.assertRaises(ValueError):
       with Pipeline() as p:
         # pylint: disable=expression-not-assigned
-        p | Create([ValueError]) | Map(raise_exception)
+        p | Create([ValueError('msg')]) | Map(raise_exception)
 
   # TODO(BEAM-1894).
   # def test_eager_pipeline(self):
diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py
index 0aef0a1791a0..86db711378be 100644
--- a/sdks/python/apache_beam/runners/common.py
+++ b/sdks/python/apache_beam/runners/common.py
@@ -20,6 +20,7 @@
 """Worker operations executor."""
 
 import sys
+import traceback
 
 from apache_beam.internal import util
 from apache_beam.metrics.execution import ScopedMetricsContainer
@@ -409,13 +410,22 @@ def finish(self):
   def _reraise_augmented(self, exn):
     if getattr(exn, '_tagged_with_step', False) or not self.step_name:
       raise
-    args = exn.args
-    if args and isinstance(args[0], str):
-      args = (args[0] + " [while running '%s']" % self.step_name,) + args[1:]
-      # Poor man's exception chaining.
-      raise type(exn), args, sys.exc_info()[2]
-    else:
-      raise
+    step_annotation = " [while running '%s']" % self.step_name
+    # To emulate exception chaining (not available in Python 2).
+    original_traceback = sys.exc_info()[2]
+    try:
+      # Attempt to construct the same kind of exception
+      # with an augmented message.
+      new_exn = type(exn)(exn.args[0] + step_annotation, *exn.args[1:])
+      new_exn._tagged_with_step = True  # Could raise attribute error.
+    except:  # pylint: disable=bare-except
+      # If anything goes wrong, construct a RuntimeError whose message
+      # records the original exception's type and message.
+      new_exn = RuntimeError(
+          traceback.format_exception_only(type(exn), exn)[-1].strip()
+          + step_annotation)
+      new_exn._tagged_with_step = True
+    raise new_exn, None, original_traceback
 
 
 class _OutputProcessor(object):
diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
index aebd2e1356db..062e6f9bef16 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
@@ -41,9 +41,9 @@ def create_pipeline(self):
     return beam.Pipeline(runner=maptask_executor_runner.MapTaskExecutorRunner())
 
   def test_assert_that(self):
-    with self.assertRaises(BeamAssertException):
+    with self.assertRaisesRegexp(BeamAssertException, 'bad_assert'):
       with self.create_pipeline() as p:
-        assert_that(p | beam.Create(['a', 'b']), equal_to(['a']))
+        assert_that(p | beam.Create(['a', 'b']), equal_to(['a']), 'bad_assert')
 
   def test_create(self):
     with self.create_pipeline() as p:
@@ -204,6 +204,21 @@ def test_windowing(self):
              | beam.Map(lambda (k, vs): (k, sorted(vs))))
       assert_that(res, equal_to([('k', [1, 2]), ('k', [100, 101, 102])]))
 
+  def test_errors(self):
+    with self.assertRaises(BaseException) as e_cm:
+      with self.create_pipeline() as p:
+        def raise_error(x):
+          raise RuntimeError('x')
+        # pylint: disable=expression-not-assigned
+        (p
+         | beam.Create(['a', 'b'])
+         | 'StageA' >> beam.Map(lambda x: x)
+         | 'StageB' >> beam.Map(lambda x: x)
+         | 'StageC' >> beam.Map(raise_error)
+         | 'StageD' >> beam.Map(lambda x: x))
+    self.assertIn('StageC', e_cm.exception.args[0])
+    self.assertNotIn('StageB', e_cm.exception.args[0])
+
 
 if __name__ == '__main__':
   logging.getLogger().setLevel(logging.INFO)

From 70216c2e542cdbb03f07e423923e5e10db48dffb Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Tue, 9 May 2017 19:07:36 -0700
Subject: [PATCH 181/387] [BEAM-2242] Ensure that jars are shaded correctly by
 running the jar plugin before the shade plugin

This required listing the maven jar plugin earlier within the file so that the jar/test jar ran before shade within the package phase.

I also update the maven shade plugin version in archetypes to pickup the fixes in the service file transformer and dropped the usage of finalName in the shade plugin so all artifacts are named the same way.
---
 examples/java/pom.xml                         | 34 -------------------
 examples/java8/pom.xml                        | 34 -------------------
 pom.xml                                       |  7 ++++
 runners/direct-java/pom.xml                   |  8 +++++
 runners/google-cloud-dataflow-java/pom.xml    |  8 +++++
 sdks/java/core/pom.xml                        |  8 +++++
 sdks/java/io/google-cloud-platform/pom.xml    |  7 ++++
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml      | 18 +++++++++-
 sdks/java/io/hbase/pom.xml                    | 25 --------------
 .../resources/archetype-resources/pom.xml     | 10 ++++--
 .../resources/archetype-resources/pom.xml     | 10 ++++--
 sdks/java/maven-archetypes/pom.xml            | 19 +++++++++++
 sdks/pom.xml                                  | 18 ----------
 13 files changed, 90 insertions(+), 116 deletions(-)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 6f0776cc1a71..8899f567bbf9 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -380,40 +380,6 @@
         </configuration>
       </plugin>
 
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <finalName>${project.artifactId}-bundled-${project.version}</finalName>
-              <artifactSet>
-                <includes>
-                  <include>*:*</include>
-                </includes>
-              </artifactSet>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/*.SF</exclude>
-                    <exclude>META-INF/*.DSA</exclude>
-                    <exclude>META-INF/*.RSA</exclude>
-                  </excludes>
-                </filter>
-              </filters>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
       <!-- Coverage analysis for unit tests. -->
       <plugin>
         <groupId>org.jacoco</groupId>
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 3c9ed71ff89f..9337e8910baa 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -178,40 +178,6 @@
         </configuration>
       </plugin>
 
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <finalName>${project.artifactId}-bundled-${project.version}</finalName>
-              <artifactSet>
-                <includes>
-                  <include>*:*</include>
-                </includes>
-              </artifactSet>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/*.SF</exclude>
-                    <exclude>META-INF/*.DSA</exclude>
-                    <exclude>META-INF/*.RSA</exclude>
-                  </excludes>
-                </filter>
-              </filters>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
       <!-- Coverage analysis for unit tests. -->
       <plugin>
         <groupId>org.jacoco</groupId>
diff --git a/pom.xml b/pom.xml
index 9ca876ebf69c..a97479c8ed2b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1418,6 +1418,7 @@
                 <goal>shade</goal>
               </goals>
               <configuration>
+                <shadeTestJar>true</shadeTestJar>
                 <artifactSet>
                   <includes>
                     <include>com.google.guava:guava</include>
@@ -1657,6 +1658,12 @@
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>build-helper-maven-plugin</artifactId>
       </plugin>
+      <!-- Ensure that the Maven jar plugin runs before the Maven
+        shade plugin by listing the plugin higher within the file. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index c115906efb7e..0daa066221a3 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -77,6 +77,13 @@
         </executions>
       </plugin>
 
+      <!-- Ensure that the Maven jar plugin runs before the Maven
+        shade plugin by listing the plugin higher within the file. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
@@ -88,6 +95,7 @@
               <goal>shade</goal>
             </goals>
             <configuration>
+              <shadeTestJar>true</shadeTestJar>
               <artifactSet>
                 <includes>
                   <include>com.google.guava:guava</include>
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 9643b69878cd..fd26ae88202e 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -173,6 +173,14 @@
           </execution>
         </executions>
       </plugin>
+
+      <!-- Ensure that the Maven jar plugin runs before the Maven
+        shade plugin by listing the plugin higher within the file. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index ef1873d2dbf1..68d76e17021e 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -121,6 +121,13 @@
         </executions>
       </plugin>
 
+      <!-- Ensure that the Maven jar plugin runs before the Maven
+        shade plugin by listing the plugin higher within the file. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
@@ -132,6 +139,7 @@
               <goal>shade</goal>
             </goals>
             <configuration>
+              <shadeTestJar>true</shadeTestJar>
               <artifactSet>
                 <includes>
                   <include>com.google.guava:guava</include>
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index f0526f01abb0..7594365bc11c 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -43,6 +43,13 @@
         </configuration>
       </plugin>
 
+      <!-- Ensure that the Maven jar plugin runs before the Maven
+        shade plugin by listing the plugin higher within the file. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
index 1c63448ed96b..56a7454a3746 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
+++ b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
@@ -71,6 +71,15 @@
           </dependency>
         </dependencies>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <!-- Don't run the tests in parallel since HIFIOWithElasticTest uses
+             a static port number which can lead to port in use errors. -->
+          <parallel>none</parallel>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
   <!--The dataflow-runner and spark-runner profiles support using those runners
@@ -162,6 +171,13 @@
       <artifactId>beam-runners-direct-java</artifactId>
       <scope>test</scope>
     </dependency>
+    <!-- We need to depend on the non test-jar artifact to get
+       the repackaged transitive dependencies on the classpath. -->
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-io-common</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-sdks-java-io-common</artifactId>
@@ -230,4 +246,4 @@
     </dependency>
   </dependencies>
 
-</project>
\ No newline at end of file
+</project>
diff --git a/sdks/java/io/hbase/pom.xml b/sdks/java/io/hbase/pom.xml
index 70ca10568a4b..746b993d6b04 100644
--- a/sdks/java/io/hbase/pom.xml
+++ b/sdks/java/io/hbase/pom.xml
@@ -48,33 +48,8 @@
             <argLine>-Dlog4j.configuration=log4j-test.properties  -XX:-UseGCOverheadLimit ${beamSurefireArgline}</argLine>
           </configuration>
         </plugin>
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-shade-plugin</artifactId>
-          <executions>
-            <execution>
-              <id>bundle-and-repackage</id>
-              <phase>package</phase>
-              <goals>
-                <goal>shade</goal>
-              </goals>
-              <configuration>
-              <!-- Disable the creation of the reduced pom to avoid an issue with the shade
-                   plugin. Ref. https://issues.apache.org/jira/browse/MSHADE-148 -->
-                <createDependencyReducedPom>false</createDependencyReducedPom>
-              </configuration>
-            </execution>
-          </executions>
-        </plugin>
       </plugins>
     </pluginManagement>
-
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-      </plugin>
-    </plugins>
   </build>
 
   <dependencies>
diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 508ff9c1738f..3c9c48020018 100644
--- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -75,6 +75,13 @@
         </dependencies>
       </plugin>
 
+      <!-- Ensure that the Maven jar plugin runs before the Maven
+        shade plugin by listing the plugin higher within the file. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
+
       <!--
         Configures `mvn package` to produce a bundled jar ("fat jar") for runners
         that require this for job submission to a cluster.
@@ -82,7 +89,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>2.4.1</version>
+        <version>3.0.0</version>
         <executions>
           <execution>
             <phase>package</phase>
@@ -90,7 +97,6 @@
               <goal>shade</goal>
             </goals>
             <configuration>
-              <finalName>${project.artifactId}-bundled-${project.version}</finalName>
               <filters>
                 <filter>
                   <artifact>*:*</artifact>
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index 511e87594a9e..00f69ffa84b6 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -75,6 +75,13 @@
         </dependencies>
       </plugin>
 
+      <!-- Ensure that the Maven jar plugin runs before the Maven
+        shade plugin by listing the plugin higher within the file. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
+
       <!--
         Configures `mvn package` to produce a bundled jar ("fat jar") for runners
         that require this for job submission to a cluster.
@@ -82,7 +89,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>2.4.1</version>
+        <version>3.0.0</version>
         <executions>
           <execution>
             <phase>package</phase>
@@ -90,7 +97,6 @@
               <goal>shade</goal>
             </goals>
             <configuration>
-              <finalName>${project.artifactId}-bundled-${project.version}</finalName>
               <filters>
                 <filter>
                   <artifact>*:*</artifact>
diff --git a/sdks/java/maven-archetypes/pom.xml b/sdks/java/maven-archetypes/pom.xml
index 0a37c5216d1f..b7fe2747daa2 100644
--- a/sdks/java/maven-archetypes/pom.xml
+++ b/sdks/java/maven-archetypes/pom.xml
@@ -69,6 +69,25 @@
         </plugin>
       </plugins>
     </pluginManagement>
+    
+    <plugins>
+      <!-- Disable the Maven jar plugin because Maven archetypes
+         are packaged using the Maven archetype packaging plugin. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>default-jar</id>
+            <phase>none</phase>
+          </execution>
+          <execution>
+            <id>default-test-jar</id>
+            <phase>none</phase>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
   </build>
 
   <profiles>
diff --git a/sdks/pom.xml b/sdks/pom.xml
index ac84471f8315..27b9610d11e6 100644
--- a/sdks/pom.xml
+++ b/sdks/pom.xml
@@ -53,24 +53,6 @@
   </profiles>
 
   <build>
-    <pluginManagement>
-      <plugins>
-        <!-- SDKs will generally offer test suites for runners, as sdks/java does. -->
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-jar-plugin</artifactId>
-          <executions>
-            <execution>
-              <id>default-test-jar</id>
-              <goals>
-                <goal>test-jar</goal>
-              </goals>
-            </execution>
-          </executions>
-        </plugin>
-      </plugins>
-    </pluginManagement>
-
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

From 86416752cd023f704ec0eefbb5ecf01f677aeee5 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 8 May 2017 20:07:29 -0700
Subject: [PATCH 182/387] Allow any throwable in PAssert to constitute adequate
 failure

Currently, some PAssert tests require an AssertionError to be thrown. This
succeeds on all runners only because many gratuitously throw AssertionError
when they don't actually know that an assertion has failed.

The spec is just that a pipeline has to fail. We don't have a good enough story
around exception propagation to have such a strict - and fake - spec. And it
isn't cross-language anyhow FWIW, looking forward to the possibility of running
a PAssert in a pipeline combining multiple SDK harnesses.
---
 .../src/test/java/org/apache/beam/sdk/testing/PAssertTest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
index 37db4ef84824..491f001a371b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java
@@ -554,7 +554,7 @@ private static Throwable runExpectingAssertionFailure(Pipeline pipeline) {
     // is first caught by JUnit and causes a test failure.
     try {
       pipeline.run();
-    } catch (AssertionError exc) {
+    } catch (Throwable exc) {
       return exc;
     }
     fail("assertion should have failed");

From 1a8a217bf8e291a2930c99dba58a305deea3270f Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Sun, 30 Apr 2017 16:08:48 -0700
Subject: [PATCH 183/387] TestDataflowRunner: throw AssertionError only when
 assertion known failed

It is quite confusing to receive an assertion error when in fact the pipeline
has crashed because of user error interacting with e.g. timers.
---
 .../runners/dataflow/TestDataflowRunner.java  | 254 +++++++++++-------
 .../dataflow/TestDataflowRunnerTest.java      |  42 +--
 2 files changed, 172 insertions(+), 124 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
index b81b48771684..1abea99fcb6f 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
@@ -29,6 +29,7 @@
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.util.List;
+import java.util.concurrent.Callable;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
 import org.apache.beam.runners.dataflow.util.MonitoringUtil;
@@ -45,8 +46,8 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * {@link TestDataflowRunner} is a pipeline runner that wraps a
- * {@link DataflowRunner} when running tests against the {@link TestPipeline}.
+ * {@link TestDataflowRunner} is a pipeline runner that wraps a {@link DataflowRunner} when running
+ * tests against the {@link TestPipeline}.
  *
  * @see TestPipeline
  */
@@ -65,16 +66,12 @@ public class TestDataflowRunner extends PipelineRunner<DataflowPipelineJob> {
     this.runner = DataflowRunner.fromOptions(options);
   }
 
-  /**
-   * Constructs a runner from the provided options.
-   */
+  /** Constructs a runner from the provided options. */
   public static TestDataflowRunner fromOptions(PipelineOptions options) {
     TestDataflowPipelineOptions dataflowOptions = options.as(TestDataflowPipelineOptions.class);
-    String tempLocation = Joiner.on("/").join(
-        dataflowOptions.getTempRoot(),
-        dataflowOptions.getJobName(),
-        "output",
-        "results");
+    String tempLocation =
+        Joiner.on("/")
+            .join(dataflowOptions.getTempRoot(), dataflowOptions.getJobName(), "output", "results");
     dataflowOptions.setTempLocation(tempLocation);
 
     return new TestDataflowRunner(
@@ -99,88 +96,115 @@ DataflowPipelineJob run(Pipeline pipeline, DataflowRunner runner) {
     final DataflowPipelineJob job;
     job = runner.run(pipeline);
 
-    LOG.info("Running Dataflow job {} with {} expected assertions.",
-        job.getJobId(), expectedNumberOfAssertions);
+    LOG.info(
+        "Running Dataflow job {} with {} expected assertions.",
+        job.getJobId(),
+        expectedNumberOfAssertions);
 
     assertThat(job, testPipelineOptions.getOnCreateMatcher());
 
-    final ErrorMonitorMessagesHandler messageHandler =
+    Boolean jobSuccess;
+    Optional<Boolean> allAssertionsPassed;
+
+    ErrorMonitorMessagesHandler messageHandler =
         new ErrorMonitorMessagesHandler(job, new MonitoringUtil.LoggingHandler());
 
-    try {
-      Optional<Boolean> result = Optional.absent();
-
-      if (options.isStreaming()) {
-        // In streaming, there are infinite retries, so rather than timeout
-        // we try to terminate early by polling and canceling if we see
-        // an error message
-        while (true) {
-          State state = job.waitUntilFinish(Duration.standardSeconds(3), messageHandler);
-          if (state != null && state.isTerminal()) {
-            break;
-          }
+    if (options.isStreaming()) {
+      jobSuccess = waitForStreamingJobTermination(job, messageHandler);
+      // No metrics in streaming
+      allAssertionsPassed = Optional.absent();
+    } else {
+      jobSuccess = waitForBatchJobTermination(job, messageHandler);
+      allAssertionsPassed = checkForPAssertSuccess(job);
+    }
 
-          if (messageHandler.hasSeenError()) {
-            if (!job.getState().isTerminal()) {
-              LOG.info("Cancelling Dataflow job {}", job.getJobId());
-              job.cancel();
-            }
-            break;
-          }
-        }
+    // If there is a certain assertion failure, throw the most precise exception we can.
+    // There are situations where the metric will not be available, but as long as we recover
+    // the actionable message from the logs it is acceptable.
+    if (!allAssertionsPassed.isPresent()) {
+      LOG.warn("Dataflow job {} did not output a success or failure metric.", job.getJobId());
+    } else if (!allAssertionsPassed.get()) {
+      throw new AssertionError(errorMessage(job, messageHandler));
+    }
 
-        // Whether we canceled or not, this gets the final state of the job or times out
-        State finalState =
-            job.waitUntilFinish(
-                Duration.standardSeconds(options.getTestTimeoutSeconds()), messageHandler);
+    // Other failures, or jobs where metrics fell through for some reason, will manifest
+    // as simply job failures.
+    if (!jobSuccess) {
+      throw new RuntimeException(errorMessage(job, messageHandler));
+    }
 
-        // Getting the final state timed out; it may not indicate a failure.
-        // This cancellation may be the second
-        if (finalState == null || finalState == State.RUNNING) {
-          LOG.info(
-              "Dataflow job {} took longer than {} seconds to complete, cancelling.",
-              job.getJobId(),
-              options.getTestTimeoutSeconds());
-          job.cancel();
-        }
+    // If there is no reason to immediately fail, run the success matcher.
+    assertThat(job, testPipelineOptions.getOnSuccessMatcher());
+    return job;
+  }
 
-        if (messageHandler.hasSeenError()) {
-          result = Optional.of(false);
-        }
-      } else {
-        job.waitUntilFinish(Duration.standardSeconds(-1), messageHandler);
-        result = checkForPAssertSuccess(job);
+  /**
+   * Return {@code true} if the job succeeded or {@code false} if it terminated in any other manner.
+   */
+  private boolean waitForStreamingJobTermination(
+      final DataflowPipelineJob job, ErrorMonitorMessagesHandler messageHandler) {
+    // In streaming, there are infinite retries, so rather than timeout
+    // we try to terminate early by polling and canceling if we see
+    // an error message
+    options.getExecutorService().submit(new CancelOnError(job, messageHandler));
+
+    // Whether we canceled or not, this gets the final state of the job or times out
+    State finalState;
+    try {
+      finalState =
+          job.waitUntilFinish(
+              Duration.standardSeconds(options.getTestTimeoutSeconds()), messageHandler);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    } catch (InterruptedException e) {
+      Thread.interrupted();
+      return false;
+    }
+
+    // Getting the final state may have timed out; it may not indicate a failure.
+    // This cancellation may be the second
+    if (finalState == null || !finalState.isTerminal()) {
+      LOG.info(
+          "Dataflow job {} took longer than {} seconds to complete, cancelling.",
+          job.getJobId(),
+          options.getTestTimeoutSeconds());
+      try {
+        job.cancel();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
       }
+      return false;
+    } else {
+      return finalState == State.DONE && !messageHandler.hasSeenError();
+    }
+  }
 
-      if (!result.isPresent()) {
-        if (options.isStreaming()) {
-          LOG.warn(
-              "Dataflow job {} did not output a success or failure metric."
-                  + " In rare situations, some PAsserts may not have run."
-                  + " This is a known limitation of Dataflow in streaming.",
-              job.getJobId());
-        } else {
-          throw new IllegalStateException(
-              String.format(
-                  "Dataflow job %s did not output a success or failure metric.", job.getJobId()));
-        }
-      } else if (!result.get()) {
-        throw new AssertionError(
-            Strings.isNullOrEmpty(messageHandler.getErrorMessage())
-                ? String.format(
-                    "Dataflow job %s terminated in state %s but did not return a failure reason.",
-                    job.getJobId(), job.getState())
-                : messageHandler.getErrorMessage());
-      } else {
-        assertThat(job, testPipelineOptions.getOnSuccessMatcher());
+  /**
+   * Return {@code true} if the job succeeded or {@code false} if it terminated in any other manner.
+   */
+  private boolean waitForBatchJobTermination(
+      DataflowPipelineJob job, ErrorMonitorMessagesHandler messageHandler) {
+    {
+      try {
+        job.waitUntilFinish(Duration.standardSeconds(-1), messageHandler);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      } catch (InterruptedException e) {
+        Thread.interrupted();
+        return false;
       }
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      throw new RuntimeException(e);
-    } catch (IOException e) {
-      throw new RuntimeException(e);
+
+      return job.getState() == State.DONE && !messageHandler.hasSeenError();
     }
-    return job;
+  }
+
+  private static String errorMessage(
+      DataflowPipelineJob job, ErrorMonitorMessagesHandler messageHandler) {
+    return Strings.isNullOrEmpty(messageHandler.getErrorMessage())
+        ? String.format(
+            "Dataflow job %s terminated in state %s but did not return a failure reason.",
+            job.getJobId(), job.getState())
+        : messageHandler.getErrorMessage();
   }
 
   @VisibleForTesting
@@ -199,19 +223,12 @@ void updatePAssertCount(Pipeline pipeline) {
    * <p>If the pipeline is not in a failed/cancelled state and no PAsserts were used within the
    * pipeline, then this method will state that all PAsserts succeeded.
    *
-   * @return Optional.of(false) if we are certain a PAssert or some other critical thing has failed,
-   *     Optional.of(true) if we are certain all PAsserts passed, and Optional.absent() if the
-   *     evidence is inconclusive.
+   * @return Optional.of(false) if we are certain a PAssert failed. Optional.of(true) if we are
+   *     certain all PAsserts passed. Optional.absent() if the evidence is inconclusive, including
+   *     when the pipeline may have failed for other reasons.
    */
   @VisibleForTesting
-  Optional<Boolean> checkForPAssertSuccess(DataflowPipelineJob job) throws IOException {
-
-    // If the job failed, this is a definite failure. We only cancel jobs when they fail.
-    State state = job.getState();
-    if (state == State.FAILED || state == State.CANCELLED) {
-      LOG.info("Dataflow job {} terminated in failure state {}", job.getJobId(), state);
-      return Optional.of(false);
-    }
+  Optional<Boolean> checkForPAssertSuccess(DataflowPipelineJob job) {
 
     JobMetrics metrics = getJobMetrics(job);
     if (metrics == null || metrics.getMetrics() == null) {
@@ -236,8 +253,12 @@ Optional<Boolean> checkForPAssertSuccess(DataflowPipelineJob job) throws IOExcep
     }
 
     if (failures > 0) {
-      LOG.info("Failure result for Dataflow job {}. Found {} success, {} failures out of "
-          + "{} expected assertions.", job.getJobId(), successes, failures,
+      LOG.info(
+          "Failure result for Dataflow job {}. Found {} success, {} failures out of "
+              + "{} expected assertions.",
+          job.getJobId(),
+          successes,
+          failures,
           expectedNumberOfAssertions);
       return Optional.of(false);
     } else if (successes >= expectedNumberOfAssertions) {
@@ -251,6 +272,16 @@ Optional<Boolean> checkForPAssertSuccess(DataflowPipelineJob job) throws IOExcep
       return Optional.of(true);
     }
 
+    // If the job failed, this is a definite failure. We only cancel jobs when they fail.
+    State state = job.getState();
+    if (state == State.FAILED || state == State.CANCELLED) {
+      LOG.info(
+          "Dataflow job {} terminated in failure state {} without reporting a failed assertion",
+          job.getJobId(),
+          state);
+      return Optional.absent();
+    }
+
     LOG.info(
         "Inconclusive results for Dataflow job {}."
             + " Found {} success, {} failures out of {} expected assertions.",
@@ -303,8 +334,10 @@ public void process(List<JobMessage> messages) {
       for (JobMessage message : messages) {
         if (message.getMessageImportance() != null
             && message.getMessageImportance().equals("JOB_MESSAGE_ERROR")) {
-          LOG.info("Dataflow job {} threw exception. Failure message was: {}",
-              job.getJobId(), message.getMessageText());
+          LOG.info(
+              "Dataflow job {} threw exception. Failure message was: {}",
+              job.getJobId(),
+              message.getMessageText());
           errorMessage.append(message.getMessageText());
           hasSeenError = true;
         }
@@ -319,4 +352,37 @@ String getErrorMessage() {
       return errorMessage.toString();
     }
   }
+
+  private static class CancelOnError implements Callable<Void> {
+
+    private final DataflowPipelineJob job;
+    private final ErrorMonitorMessagesHandler messageHandler;
+
+    public CancelOnError(DataflowPipelineJob job, ErrorMonitorMessagesHandler messageHandler) {
+      this.job = job;
+      this.messageHandler = messageHandler;
+    }
+
+    @Override
+    public Void call() throws Exception {
+      while (true) {
+        State jobState = job.getState();
+
+        // If we see an error, cancel and note failure
+        if (messageHandler.hasSeenError()) {
+          if (!job.getState().isTerminal()) {
+            job.cancel();
+            LOG.info("Cancelling Dataflow job {}", job.getJobId());
+            return null;
+          }
+        }
+
+        if (jobState.isTerminal()) {
+          return null;
+        }
+
+        Thread.sleep(3000L);
+      }
+    }
+  }
 }
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
index 883d3443d75f..1c0876a5f8a4 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
@@ -124,6 +124,10 @@ public void testRunBatchJobThatSucceeds() throws Exception {
     assertEquals(mockJob, runner.run(p, mockRunner));
   }
 
+  /**
+   * Tests that when a batch job terminates in a failure state even if all assertions
+   * passed, it throws an error to that effect.
+   */
   @Test
   public void testRunBatchJobThatFails() throws Exception {
     Pipeline p = TestPipeline.create(options);
@@ -140,12 +144,9 @@ public void testRunBatchJobThatFails() throws Exception {
 
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
     when(mockClient.getJobMetrics(anyString()))
-        .thenReturn(generateMockMetricResponse(false /* success */, false /* tentative */));
-    try {
-      runner.run(p, mockRunner);
-    } catch (AssertionError expected) {
-      return;
-    }
+        .thenReturn(generateMockMetricResponse(true /* success */, false /* tentative */));
+    expectedException.expect(RuntimeException.class);
+    runner.run(p, mockRunner);
     // Note that fail throws an AssertionError which is why it is placed out here
     // instead of inside the try-catch block.
     fail("AssertionError expected");
@@ -356,22 +357,6 @@ public void testCheckingForSuccessSkipsNonTentativeMetrics() throws Exception {
     assertThat(runner.checkForPAssertSuccess(job), equalTo(Optional.<Boolean>absent()));
   }
 
-  /**
-   * Tests that if a streaming pipeline terminates with FAIL that the check for PAssert
-   * success is a conclusive failure.
-   */
-  @Test
-  public void testStreamingPipelineFailsIfServiceFails() throws Exception {
-    DataflowPipelineJob job = spy(new DataflowPipelineJob(mockClient, "test-job", options, null));
-    Pipeline p = TestPipeline.create(options);
-    PCollection<Integer> pc = p.apply(Create.of(1, 2, 3));
-    PAssert.that(pc).containsInAnyOrder(1, 2, 3);
-
-    TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
-    doReturn(State.FAILED).when(job).getState();
-    assertThat(runner.checkForPAssertSuccess(job), equalTo(Optional.of(false)));
-  }
-
   /**
    * Tests that if a streaming pipeline crash loops for a non-assertion reason that the test run
    * throws an {@link AssertionError}.
@@ -411,12 +396,8 @@ public State answer(InvocationOnMock invocation) {
         .thenReturn(generateMockMetricResponse(false /* success */, true /* tentative */));
     TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient);
 
-    try {
-      runner.run(pipeline, mockRunner);
-    } catch (AssertionError exc) {
-      return;
-    }
-    fail("AssertionError expected");
+    expectedException.expect(RuntimeException.class);
+    runner.run(pipeline, mockRunner);
   }
 
   @Test
@@ -581,7 +562,7 @@ public void testBatchOnSuccessMatcherWhenPipelineFails() throws Exception {
    * Tests that when a streaming pipeline terminates in FAIL that the {@link
    * TestPipelineOptions#setOnSuccessMatcher(SerializableMatcher) on success matcher} is not
    * invoked.
-   */
+   */t ad
   @Test
   public void testStreamingOnSuccessMatcherWhenPipelineFails() throws Exception {
     options.setStreaming(true);
@@ -603,8 +584,9 @@ public void testStreamingOnSuccessMatcherWhenPipelineFails() throws Exception {
     when(mockJob.waitUntilFinish(any(Duration.class), any(JobMessagesHandler.class)))
         .thenReturn(State.FAILED);
 
+    expectedException.expect(RuntimeException.class);
     runner.run(p, mockRunner);
-    // If the onSuccessMatcher were invoked, it would have crashed here.
+    // If the onSuccessMatcher were invoked, it would have crashed here with AssertionError
   }
 
   static class TestSuccessMatcher extends BaseMatcher<PipelineResult> implements

From b79c888b01d0c42a35ecf1b98134261237e9c83e Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 14 Feb 2017 14:54:11 -0800
Subject: [PATCH 184/387] Re-enable UsesTimersInParDo tests in Dataflow runner

---
 runners/google-cloud-dataflow-java/pom.xml                   | 1 -
 .../test/java/org/apache/beam/sdk/transforms/ParDoTest.java  | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index fd26ae88202e..380933576cdc 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -164,7 +164,6 @@
                 org.apache.beam.sdk.testing.UsesGaugeMetrics,
                 org.apache.beam.sdk.testing.UsesSetState,
                 org.apache.beam.sdk.testing.UsesMapState,
-                org.apache.beam.sdk.testing.UsesTimersInParDo,
                 org.apache.beam.sdk.testing.UsesSplittableParDo,
                 org.apache.beam.sdk.testing.UsesUnboundedPCollections,
                 org.apache.beam.sdk.testing.UsesTestStream,
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index ef27f4c03446..35c02ba8c9d0 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -54,7 +54,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
@@ -2745,7 +2744,7 @@ public void onTimer(OnTimerContext context) {}
         };
 
     PCollection<Integer> output = pipeline.apply(Create.of(KV.of("hello", 37))).apply(ParDo.of(fn));
-    thrown.expect(PipelineExecutionException.class);
+    thrown.expect(RuntimeException.class);
     // Note that runners can reasonably vary their message - this matcher should be flexible
     // and can be evolved.
     thrown.expectMessage("relative timers");
@@ -2775,7 +2774,7 @@ public void onTimer(OnTimerContext context) {}
         };
 
     PCollection<Integer> output = pipeline.apply(Create.of(KV.of("hello", 37))).apply(ParDo.of(fn));
-    thrown.expect(PipelineExecutionException.class);
+    thrown.expect(RuntimeException.class);
     // Note that runners can reasonably vary their message - this matcher should be flexible
     // and can be evolved.
     thrown.expectMessage("event time timer");

From 22d87785cf1bb32af31628813f20a2d2ad854dcd Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Wed, 10 May 2017 15:19:50 -0700
Subject: [PATCH 185/387] Fix a typo in TestDataflowRunnerTest

---
 .../apache/beam/runners/dataflow/TestDataflowRunnerTest.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
index 1c0876a5f8a4..15faf2612c9c 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java
@@ -562,7 +562,7 @@ public void testBatchOnSuccessMatcherWhenPipelineFails() throws Exception {
    * Tests that when a streaming pipeline terminates in FAIL that the {@link
    * TestPipelineOptions#setOnSuccessMatcher(SerializableMatcher) on success matcher} is not
    * invoked.
-   */t ad
+   */
   @Test
   public void testStreamingOnSuccessMatcherWhenPipelineFails() throws Exception {
     options.setStreaming(true);

From 2c7abaca1112e56290a7ecb886d59805df5f69ee Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Wed, 10 May 2017 14:29:45 -0700
Subject: [PATCH 186/387] Skip generating empty jars for parent poms

---
 pom.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pom.xml b/pom.xml
index a97479c8ed2b..e7eb729b6d35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1195,6 +1195,9 @@
               </goals>
             </execution>
           </executions>
+          <configuration>
+            <skipIfEmpty>true</skipIfEmpty>
+          </configuration>
         </plugin>
 
         <plugin>

From 5a4e13149fb9648cb19428e8eef1fdddf2169785 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Wed, 10 May 2017 08:56:54 -0700
Subject: [PATCH 187/387] Update SDK Coders to return the Empty List from
 getCoderArguments

Brings existing Coders in line with the documentation
---
 .../java/org/apache/beam/sdk/coders/AvroCoder.java   |  5 -----
 .../org/apache/beam/sdk/coders/ByteArrayCoder.java   |  6 ------
 .../apache/beam/sdk/transforms/join/UnionCoder.java  |  3 ++-
 .../java/org/apache/beam/sdk/util/WindowedValue.java | 12 ++++++++++--
 .../apache/beam/sdk/transforms/CombineFnsTest.java   |  3 ++-
 5 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
index bba669de2302..94b4a9ffabcd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java
@@ -318,11 +318,6 @@ public T decode(InputStream inStream) throws IOException {
     return reader.get().read(null, decoderInstance);
   }
 
-  @Override
-  public List<? extends Coder<?>> getCoderArguments() {
-    return null;
-  }
-
   /**
    * @throws NonDeterministicException when the type may not be deterministically
    * encoded using the given {@link Schema}, the {@code directBinaryEncoder}, and the
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
index 3b3838823814..16094ddd88a4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java
@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.List;
 import org.apache.beam.sdk.util.ExposedByteArrayOutputStream;
 import org.apache.beam.sdk.util.StreamUtils;
 import org.apache.beam.sdk.util.VarInt;
@@ -112,11 +111,6 @@ public byte[] decode(InputStream inStream, Context context)
     }
   }
 
-  @Override
-  public List<? extends Coder<?>> getCoderArguments() {
-    return null;
-  }
-
   @Override
   public void verifyDeterministic() {}
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
index 66959d34aa47..6590ca08338c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/UnionCoder.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Collections;
 import java.util.List;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
@@ -94,7 +95,7 @@ public RawUnionValue decode(InputStream inStream, Context context)
 
   @Override
   public List<? extends Coder<?>> getCoderArguments() {
-    return null;
+    return Collections.emptyList();
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
index 444521adc8f7..830095431085 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java
@@ -685,9 +685,17 @@ public void registerByteSizeObserver(WindowedValue<T> value,
       valueCoder.registerByteSizeObserver(value.getValue(), observer);
     }
 
+    /**
+     * {@inheritDoc}.
+     *
+     * @return a singleton list containing the {@code valueCoder} of this
+     *         {@link FullWindowedValueCoder}.
+     */
     @Override
     public List<? extends Coder<?>> getCoderArguments() {
-      return null;
+      // The value type is the only generic type parameter exposed by this coder. The component
+      // coders include the window coder as well
+      return Collections.singletonList(valueCoder);
     }
 
     @Override
@@ -758,7 +766,7 @@ public void registerByteSizeObserver(
 
     @Override
     public List<? extends Coder<?>> getCoderArguments() {
-      return Arrays.<Coder<?>>asList(valueCoder);
+      return Collections.singletonList(valueCoder);
     }
   }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
index 33c652a523af..aeb769264951 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java
@@ -27,6 +27,7 @@
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
@@ -360,7 +361,7 @@ public UserString decode(InputStream inStream, Context context)
 
     @Override
     public List<? extends Coder<?>> getCoderArguments() {
-      return null;
+      return Collections.emptyList();
     }
 
     @Override

From 968a59f7c77fd35757aad0c7869292c80684d21a Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 10 May 2017 16:41:00 -0700
Subject: [PATCH 188/387] Correct javadoc for mobile gaming examples

---
 .../apache/beam/examples/complete/game/GameStats.java  |  8 ++++----
 .../beam/examples/complete/game/HourlyTeamScore.java   |  5 ++---
 .../beam/examples/complete/game/LeaderBoard.java       | 10 +++++-----
 .../apache/beam/examples/complete/game/UserScore.java  | 10 ++++------
 4 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
index abbb13bf93ee..dbc7f88e9974 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
@@ -78,17 +78,17 @@
  * <p>Run {@code org.apache.beam.examples.complete.game.injector.Injector} to generate
  * pubsub data for this pipeline. The {@code Injector} documentation provides more detail.
  *
- * <p>To execute this pipeline using the Dataflow service, specify the pipeline configuration
- * like this:
+ * <p>To execute this pipeline, specify the pipeline configuration like this:
  * <pre>{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowRunner
+ *   --runner=YOUR_RUNNER
  *   --dataset=YOUR-DATASET
  *   --topic=projects/YOUR-PROJECT/topics/YOUR-TOPIC
  * }
  * </pre>
- * where the BigQuery dataset you specify must already exist. The PubSub topic you specify should
+ *
+ * <p>The BigQuery dataset you specify must already exist. The PubSub topic you specify should
  * be the same topic to which the Injector is publishing.
  */
 public class GameStats extends LeaderBoard {
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
index 6a322da8fc87..bab74cefe7a4 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
@@ -53,13 +53,12 @@
  * {@link UserScore} pipeline. However, our batch processing is high-latency, in that we don't get
  * results from plays at the beginning of the batch's time period until the batch is processed.
  *
- * <p>To execute this pipeline using the Dataflow service, specify the pipeline configuration
- * like this:
+ * <p>To execute this pipeline, specify the pipeline configuration like this:
  * <pre>{@code
- *   --project=YOUR_PROJECT_ID
  *   --tempLocation=YOUR_TEMP_DIRECTORY
  *   --runner=YOUR_RUNNER
  *   --output=YOUR_OUTPUT_DIRECTORY
+ *   (possibly options specific to your runner or permissions for your temp/output locations)
  * }
  * </pre>
  *
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
index f673a8de688a..b879b7037d21 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
@@ -77,18 +77,18 @@
  * <p>Run {@code injector.Injector} to generate pubsub data for this pipeline.  The Injector
  * documentation provides more detail on how to do this.
  *
- * <p>To execute this pipeline using the Dataflow service, specify the pipeline configuration
- * like this:
+ * <p>To execute this pipeline, specify the pipeline configuration like this:
  * <pre>{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowRunner
+ *   --runner=YOUR_RUNNER
  *   --dataset=YOUR-DATASET
  *   --topic=projects/YOUR-PROJECT/topics/YOUR-TOPIC
  * }
  * </pre>
- * where the BigQuery dataset you specify must already exist.
- * The PubSub topic you specify should be the same topic to which the Injector is publishing.
+ *
+ * <p>The BigQuery dataset you specify must already exist. The PubSub topic you specify should be
+ * the same topic to which the Injector is publishing.
  */
 public class LeaderBoard extends HourlyTeamScore {
 
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index 7297bcdf3e94..ee7b79219b14 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -45,8 +45,8 @@
 
 /**
  * This class is the first in a series of four pipelines that tell a story in a 'gaming' domain.
- * Concepts: batch processing; reading input from Google Cloud Storage and writing output to
- * BigQuery; using standalone DoFns; use of the sum by key transform; examples of
+ * Concepts: batch processing, reading input from text files, writing output to
+ * text files, using standalone DoFns, use of the sum per key transform, and use of
  * Java 8 lambda syntax.
  *
  * <p>In this gaming scenario, many users play, as members of different teams, over the course of a
@@ -57,16 +57,14 @@
  * sum of scores per user, over an entire batch of gaming data (collected, say, for each day). The
  * batch processing will not include any late data that arrives after the day's cutoff point.
  *
- * <p>To execute this pipeline using the Dataflow service and static example input data, specify
- * the pipeline configuration like this:
+ * <p>To execute this pipeline, specify the pipeline configuration like this:
  * <pre>{@code
- *   --project=YOUR_PROJECT_ID
  *   --tempLocation=YOUR_TEMP_DIRECTORY
  *   --runner=YOUR_RUNNER
  *   --output=YOUR_OUTPUT_DIRECTORY
+ *   (possibly options specific to your runner or permissions for your temp/output locations)
  * }
  * </pre>
- * where the BigQuery dataset you specify must already exist.
  *
  * <p>Optionally include the --input argument to specify a batch input file.
  * See the --input default value for example batch data file, or use {@code injector.Injector} to

From 8cd98bd968740e6f37e8ec9cf2d273c9f948698b Mon Sep 17 00:00:00 2001
From: bchambers <bchambers@google.com>
Date: Tue, 9 May 2017 15:45:50 -0700
Subject: [PATCH 189/387] [BEAM-2244] Move details of Metrics to Runners Core

Largeish changes this required were:

  - splitting the MetricsContainer into an interface in Java Core with
    an implementation in Runners Core
  - modifying the various *Cell classes to have a name
  - cleaning up dependency cross-fire.
---
 .../core}/metrics/MetricFiltering.java        |   5 +-
 .../beam/runners/core}/metrics/MetricKey.java |   4 +-
 .../runners/core/metrics/package-info.java    |  22 ++
 .../core}/metrics/MetricFilteringTest.java    |   5 +-
 .../beam/runners/core/LateDataUtils.java      |   4 +-
 .../runners/core}/metrics/CounterCell.java    |  36 +++-
 .../runners/core}/metrics/DirtyState.java     |   2 +-
 .../core}/metrics/DistributionCell.java       |  28 ++-
 .../core}/metrics/DistributionData.java       |   4 +-
 .../beam/runners/core}/metrics/GaugeCell.java |  36 ++--
 .../beam/runners/core}/metrics/GaugeData.java |   4 +-
 .../runners/core}/metrics/MetricCell.java     |  17 +-
 .../runners/core}/metrics/MetricUpdates.java  |   3 +-
 .../core/metrics/MetricsContainerImpl.java    | 188 ++++++++++++++++++
 .../metrics/MetricsContainerStepMap.java      |  32 +--
 .../runners/core}/metrics/MetricsMap.java     |   3 +-
 .../runners/core/metrics/package-info.java    |  22 ++
 .../core/LateDataDroppingDoFnRunnerTest.java  |   9 +-
 .../beam/runners/core/ReduceFnRunnerTest.java |  34 ++--
 .../runners/core/StatefulDoFnRunnerTest.java  |  10 +-
 .../core}/metrics/CounterCellTest.java        |  11 +-
 .../runners/core}/metrics/DirtyStateTest.java |   2 +-
 .../core}/metrics/DistributionCellTest.java   |   5 +-
 .../runners/core}/metrics/GaugeCellTest.java  |   6 +-
 .../core/metrics/MetricUpdateMatchers.java    |  82 ++++++++
 .../metrics/MetricsContainerImplTest.java     |  29 +--
 .../metrics/MetricsContainerStepMapTest.java  | 104 +++++-----
 .../runners/core}/metrics/MetricsMapTest.java |   2 +-
 .../beam/runners/direct/DirectMetrics.java    |  14 +-
 .../runners/direct/StepTransformResult.java   |   2 +-
 .../runners/direct/TransformExecutor.java     |  10 +-
 .../beam/runners/direct/TransformResult.java  |   2 +-
 .../runners/direct/DirectMetricsTest.java     |  14 +-
 .../beam/runners/flink/FlinkRunnerResult.java |   4 +-
 .../metrics/DoFnRunnerWithMetricsUpdate.java  |   3 +-
 .../flink/metrics/FlinkMetricContainer.java   |   7 +-
 .../flink/metrics/MetricsAccumulator.java     |   2 +-
 .../flink/metrics/ReaderInvocationUtil.java   |   5 +-
 runners/google-cloud-dataflow-java/pom.xml    |   2 +-
 .../runners/dataflow/DataflowMetrics.java     |   4 +-
 .../runners/dataflow/DataflowMetricsTest.java |   4 +-
 .../runners/spark/SparkPipelineResult.java    |   2 +-
 .../beam/runners/spark/io/SourceRDD.java      |   2 +-
 .../spark/io/SparkUnboundedSource.java        |   2 +-
 .../spark/metrics/MetricsAccumulator.java     |   2 +-
 .../metrics/MetricsAccumulatorParam.java      |   2 +-
 .../spark/metrics/SparkBeamMetric.java        |   4 +-
 .../SparkGroupAlsoByWindowViaWindowSet.java   |  10 +-
 .../spark/stateful/StateSpecFunctions.java    |   2 +-
 .../translation/DoFnRunnerWithMetrics.java    |   5 +-
 .../spark/translation/MultiDoFnFunction.java  |   2 +-
 .../translation/TransformTranslator.java      |   2 +-
 .../StreamingTransformTranslator.java         |   2 +-
 .../spark/io/ReaderToIteratorAdapterTest.java |   4 +-
 .../ResumeFromCheckpointStreamingTest.java    |   2 +-
 .../streaming/StreamingSourceMetricsTest.java |   2 +-
 .../beam/sdk/metrics/DistributionResult.java  |   3 +
 .../apache/beam/sdk/metrics/GaugeResult.java  |   3 +
 .../org/apache/beam/sdk/metrics/Metric.java   |   7 +-
 .../org/apache/beam/sdk/metrics/Metrics.java  |   8 +-
 .../beam/sdk/metrics/MetricsContainer.java    | 146 +-------------
 .../beam/sdk/metrics/MetricsEnvironment.java  |   9 +-
 .../apache/beam/sdk/metrics/SinkMetrics.java  |   4 +
 .../beam/sdk/metrics/SourceMetrics.java       |   3 +
 ...tchers.java => MetricResultsMatchers.java} |  58 +-----
 .../sdk/metrics/MetricsEnvironmentTest.java   |  23 ++-
 .../apache/beam/sdk/metrics/MetricsTest.java  |  43 ++--
 .../apache/beam/sdk/io/kafka/KafkaIOTest.java |  54 +++--
 68 files changed, 706 insertions(+), 482 deletions(-)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-construction-java/src/main/java/org/apache/beam/runners/core}/metrics/MetricFiltering.java (95%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-construction-java/src/main/java/org/apache/beam/runners/core}/metrics/MetricKey.java (94%)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => runners/core-construction-java/src/test/java/org/apache/beam/runners/core}/metrics/MetricFilteringTest.java (96%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/CounterCell.java (77%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/DirtyState.java (98%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/DistributionCell.java (81%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/DistributionData.java (95%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/GaugeCell.java (78%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/GaugeData.java (95%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/MetricCell.java (78%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/MetricUpdates.java (98%)
 create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/MetricsContainerStepMap.java (93%)
 rename {sdks/java/core/src/main/java/org/apache/beam/sdk => runners/core-java/src/main/java/org/apache/beam/runners/core}/metrics/MetricsMap.java (98%)
 create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => runners/core-java/src/test/java/org/apache/beam/runners/core}/metrics/CounterCellTest.java (88%)
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => runners/core-java/src/test/java/org/apache/beam/runners/core}/metrics/DirtyStateTest.java (97%)
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => runners/core-java/src/test/java/org/apache/beam/runners/core}/metrics/DistributionCellTest.java (90%)
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => runners/core-java/src/test/java/org/apache/beam/runners/core}/metrics/GaugeCellTest.java (90%)
 create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricUpdateMatchers.java
 rename sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java => runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java (88%)
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => runners/core-java/src/test/java/org/apache/beam/runners/core}/metrics/MetricsContainerStepMapTest.java (66%)
 rename {sdks/java/core/src/test/java/org/apache/beam/sdk => runners/core-java/src/test/java/org/apache/beam/runners/core}/metrics/MetricsMapTest.java (98%)
 rename sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/{MetricMatchers.java => MetricResultsMatchers.java} (78%)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricFiltering.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricFiltering.java
similarity index 95%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricFiltering.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricFiltering.java
index a3e43e16a018..d469d2054f5e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricFiltering.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricFiltering.java
@@ -16,10 +16,13 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
 import com.google.common.base.Objects;
 import java.util.Set;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.MetricsFilter;
 
 /**
  * Implements matching for metrics filters. Specifically, matching for metric name,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricKey.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricKey.java
similarity index 94%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricKey.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricKey.java
index 8706853f4871..58d40553b4ea 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricKey.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/MetricKey.java
@@ -15,12 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import com.google.auto.value.AutoValue;
 import java.io.Serializable;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.metrics.MetricName;
 
 /**
  * Metrics are keyed by the step name they are associated with and the name of the metric.
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
new file mode 100644
index 000000000000..263a705ac033
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for runners to implement metrics.
+ */
+package org.apache.beam.runners.core.metrics;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricFilteringTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/metrics/MetricFilteringTest.java
similarity index 96%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricFilteringTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/metrics/MetricFilteringTest.java
index dc2fa0aa6e5c..8953f215f1dc 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricFilteringTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/metrics/MetricFilteringTest.java
@@ -16,13 +16,16 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.util.HashSet;
 import java.util.Set;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
index f7c0d31374ea..732e60c6c657 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
@@ -22,7 +22,7 @@
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Iterables;
 import javax.annotation.Nullable;
-import org.apache.beam.sdk.metrics.CounterCell;
+import org.apache.beam.runners.core.metrics.CounterCell;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -71,7 +71,7 @@ public boolean apply(@Nullable WindowedValue<V> input) {
                         .isBefore(timerInternals.currentInputWatermarkTime());
                 if (expired) {
                   // The element is too late for this window.
-                  droppedDueToLateness.update(1L);
+                  droppedDueToLateness.inc();
                   WindowTracing.debug(
                       "GroupAlsoByWindow: Dropping element at {} for key: {}; "
                           + "window: {} since it is too far behind inputWatermark: {}",
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/CounterCell.java
similarity index 77%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/CounterCell.java
index 4b8548f7da22..4378bb9c7b8a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/CounterCell.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/CounterCell.java
@@ -15,11 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.MetricName;
 
 /**
  * Tracks the current value (and delta) for a Counter metric for a specific context and bundle.
@@ -30,34 +33,40 @@
  * indirection.
  */
 @Experimental(Kind.METRICS)
-public class CounterCell implements MetricCell<Counter, Long> {
+public class CounterCell implements Counter, MetricCell<Long> {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicLong value = new AtomicLong();
+  private final MetricName name;
 
   /**
    * Package-visibility because all {@link CounterCell CounterCells} should be created by
-   * {@link MetricsContainer#getCounter(MetricName)}.
+   * {@link MetricsContainerImpl#getCounter(MetricName)}.
    */
-  CounterCell() {}
+  CounterCell(MetricName name) {
+    this.name = name;
+  }
 
   /**
    * Increment the counter by the given amount.
    * @param n value to increment by. Can be negative to decrement.
    */
-  public void update(long n) {
+  @Override
+  public void inc(long n) {
     value.addAndGet(n);
     dirty.afterModification();
   }
 
-  @Override
-  public void update(Long n) {
-    throw new UnsupportedOperationException("CounterCell.update(Long n) should not be used"
-    + " as it performs unnecessary boxing/unboxing. Use CounterCell.update(long n) instead.");
+  public void inc() {
+    inc(1);
   }
 
-  @Override public void update(MetricCell<Counter, Long> other) {
-    update((long) other.getCumulative());
+  public void dec() {
+    inc(-1);
+  }
+
+  public void dec(long n) {
+    inc(-1 * n);
   }
 
   @Override
@@ -69,4 +78,9 @@ public DirtyState getDirty() {
   public Long getCumulative() {
     return value.get();
   }
+
+  @Override
+  public MetricName getName() {
+    return name;
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DirtyState.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DirtyState.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DirtyState.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DirtyState.java
index 4e0c15cb2d67..532fc2a36f9b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DirtyState.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DirtyState.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
 import java.io.Serializable;
 import java.util.concurrent.atomic.AtomicReference;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionCell.java
similarity index 81%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionCell.java
index 93a3649d19a5..5a5099a95cc7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionCell.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionCell.java
@@ -15,11 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.metrics.Distribution;
+import org.apache.beam.sdk.metrics.MetricName;
 
 /**
  * Tracks the current value (and delta) for a Distribution metric.
@@ -30,25 +33,28 @@
  * of indirection.
  */
 @Experimental(Kind.METRICS)
-public class DistributionCell implements MetricCell<Distribution, DistributionData> {
+public class DistributionCell implements Distribution, MetricCell<DistributionData> {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicReference<DistributionData> value =
       new AtomicReference<>(DistributionData.EMPTY);
+  private final MetricName name;
 
   /**
    * Package-visibility because all {@link DistributionCell DistributionCells} should be created by
-   * {@link MetricsContainer#getDistribution(MetricName)}.
+   * {@link MetricsContainerImpl#getDistribution(MetricName)}.
    */
-  DistributionCell() {}
+  DistributionCell(MetricName name) {
+    this.name = name;
+  }
 
   /** Increment the distribution by the given amount. */
+  @Override
   public void update(long n) {
     update(DistributionData.singleton(n));
   }
 
-  @Override
-  public void update(DistributionData data) {
+  void update(DistributionData data) {
     DistributionData original;
     do {
       original = value.get();
@@ -56,11 +62,6 @@ public void update(DistributionData data) {
     dirty.afterModification();
   }
 
-  @Override
-  public void update(MetricCell<Distribution, DistributionData> other) {
-    update(other.getCumulative());
-  }
-
   @Override
   public DirtyState getDirty() {
     return dirty;
@@ -70,5 +71,10 @@ public DirtyState getDirty() {
   public DistributionData getCumulative() {
     return value.get();
   }
+
+  @Override
+  public MetricName getName() {
+    return name;
+  }
 }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionData.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionData.java
similarity index 95%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionData.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionData.java
index 8068e1b0dc6b..099e63fb41c9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionData.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DistributionData.java
@@ -15,10 +15,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import com.google.auto.value.AutoValue;
 import java.io.Serializable;
+import org.apache.beam.sdk.metrics.DistributionResult;
 
 /**
  * Data describing the the distribution. This should retain enough detail that it can be combined
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeCell.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeCell.java
similarity index 78%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeCell.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeCell.java
index 0cdd56891718..795e8265e024 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeCell.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeCell.java
@@ -15,10 +15,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.metrics.Gauge;
+import org.apache.beam.sdk.metrics.MetricName;
 
 /**
  * Tracks the current value (and delta) for a {@link Gauge} metric.
@@ -29,18 +32,27 @@
  * of indirection.
  */
 @Experimental(Experimental.Kind.METRICS)
-public class GaugeCell implements MetricCell<Gauge, GaugeData> {
+public class GaugeCell implements Gauge, MetricCell<GaugeData> {
 
   private final DirtyState dirty = new DirtyState();
   private final AtomicReference<GaugeData> gaugeValue = new AtomicReference<>(GaugeData.empty());
+  private final MetricName name;
+
+  /**
+   * Package-visibility because all {@link GaugeCell GaugeCells} should be created by
+   * {@link MetricsContainerImpl#getGauge(MetricName)}.
+   */
+  GaugeCell(MetricName name) {
+    this.name = name;
+  }
 
   /** Set the gauge to the given value. */
+  @Override
   public void set(long value) {
     update(GaugeData.create(value));
   }
 
-  @Override
-  public void update(GaugeData data) {
+  void update(GaugeData data) {
     GaugeData original;
     do {
       original = gaugeValue.get();
@@ -48,16 +60,6 @@ public void update(GaugeData data) {
     dirty.afterModification();
   }
 
-  @Override
-  public void update(MetricCell<Gauge, GaugeData> other) {
-    GaugeData original;
-    do {
-      original = gaugeValue.get();
-    } while (!gaugeValue.compareAndSet(original, original.combine(other.getCumulative())));
-    dirty.afterModification();
-    update(other.getCumulative());
-  }
-
   @Override
   public DirtyState getDirty() {
     return dirty;
@@ -67,4 +69,10 @@ public DirtyState getDirty() {
   public GaugeData getCumulative() {
     return gaugeValue.get();
   }
+
+
+  @Override
+  public MetricName getName() {
+    return name;
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeData.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeData.java
similarity index 95%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeData.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeData.java
index bf3401d38b90..15a353f78e46 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeData.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/GaugeData.java
@@ -15,10 +15,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import com.google.auto.value.AutoValue;
 import java.io.Serializable;
+import org.apache.beam.sdk.metrics.GaugeResult;
 import org.joda.time.Instant;
 
 /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricCell.java
similarity index 78%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricCell.java
index 403cac208319..2b773f05c4ef 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricCell.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricCell.java
@@ -15,7 +15,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import java.io.Serializable;
 import org.apache.beam.sdk.annotations.Experimental;
@@ -25,22 +26,10 @@
  * A {@link MetricCell} is used for accumulating in-memory changes to a metric. It represents a
  * specific metric name in a single context.
  *
- * @param <UserT> The type of the user interface for reporting changes to this cell.
  * @param <DataT> The type of metric data stored (and extracted) from this cell.
  */
 @Experimental(Kind.METRICS)
-public interface MetricCell<UserT extends Metric, DataT> extends Serializable {
-
-  /**
-   * Update value of this cell.
-   */
-  void update(DataT data);
-
-  /**
-   * Update value of this cell by merging the value of another cell.
-   */
-  void update(MetricCell<UserT, DataT> other);
-
+public interface MetricCell<DataT> extends Serializable {
   /**
    * Return the {@link DirtyState} tracking whether this metric cell contains uncommitted changes.
    */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricUpdates.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricUpdates.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java
index 9cf6a5ca2bf5..eae3305bd268 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricUpdates.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java
@@ -15,7 +15,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.Iterables;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
new file mode 100644
index 000000000000..6967bf0d85b9
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.metrics;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.metrics.Metric;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricsContainer;
+
+/**
+ * Holds the metrics for a single step and unit-of-commit (bundle).
+ *
+ * <p>This class is thread-safe. It is intended to be used with 1 (or more) threads are updating
+ * metrics and at-most 1 thread is extracting updates by calling {@link #getUpdates} and
+ * {@link #commitUpdates}. Outside of this it is still safe. Although races in the update extraction
+ * may cause updates that don't actually have any changes, it will never lose an update.
+ *
+ * <p>For consistency, all threads that update metrics should finish before getting the final
+ * cumulative values/updates.
+ */
+@Experimental(Kind.METRICS)
+public class MetricsContainerImpl implements Serializable, MetricsContainer {
+
+  private final String stepName;
+
+  private MetricsMap<MetricName, CounterCell> counters =
+      new MetricsMap<>(new MetricsMap.Factory<MetricName, CounterCell>() {
+        @Override
+        public CounterCell createInstance(MetricName name) {
+          return new CounterCell(name);
+        }
+      });
+
+  private MetricsMap<MetricName, DistributionCell> distributions =
+      new MetricsMap<>(new MetricsMap.Factory<MetricName, DistributionCell>() {
+        @Override
+        public DistributionCell createInstance(MetricName name) {
+          return new DistributionCell(name);
+        }
+      });
+
+  private MetricsMap<MetricName, GaugeCell> gauges =
+      new MetricsMap<>(new MetricsMap.Factory<MetricName, GaugeCell>() {
+        @Override
+        public GaugeCell createInstance(MetricName name) {
+          return new GaugeCell(name);
+        }
+      });
+
+  /**
+   * Create a new {@link MetricsContainerImpl} associated with the given {@code stepName}.
+   */
+  public MetricsContainerImpl(String stepName) {
+    this.stepName = stepName;
+  }
+
+  @Override
+  public CounterCell getCounter(MetricName metricName) {
+    return counters.get(metricName);
+  }
+
+  @Override
+  public DistributionCell getDistribution(MetricName metricName) {
+    return distributions.get(metricName);
+  }
+
+  @Override
+  public GaugeCell getGauge(MetricName metricName) {
+    return gauges.get(metricName);
+  }
+
+  private <UpdateT, CellT extends MetricCell<UpdateT>>
+  ImmutableList<MetricUpdate<UpdateT>> extractUpdates(MetricsMap<MetricName, CellT> cells) {
+    ImmutableList.Builder<MetricUpdate<UpdateT>> updates = ImmutableList.builder();
+    for (Map.Entry<MetricName, CellT> cell : cells.entries()) {
+      if (cell.getValue().getDirty().beforeCommit()) {
+        updates.add(MetricUpdate.create(MetricKey.create(stepName, cell.getKey()),
+            cell.getValue().getCumulative()));
+      }
+    }
+    return updates.build();
+  }
+
+  /**
+   * Return the cumulative values for any metrics that have changed since the last time updates were
+   * committed.
+   */
+  public MetricUpdates getUpdates() {
+    return MetricUpdates.create(
+        extractUpdates(counters),
+        extractUpdates(distributions),
+        extractUpdates(gauges));
+  }
+
+  private void commitUpdates(MetricsMap<MetricName, ? extends MetricCell<?>> cells) {
+    for (MetricCell<?> cell : cells.values()) {
+      cell.getDirty().afterCommit();
+    }
+  }
+
+  /**
+   * Mark all of the updates that were retrieved with the latest call to {@link #getUpdates()} as
+   * committed.
+   */
+  public void commitUpdates() {
+    commitUpdates(counters);
+    commitUpdates(distributions);
+    commitUpdates(gauges);
+  }
+
+  private <UserT extends Metric, UpdateT, CellT extends MetricCell<UpdateT>>
+  ImmutableList<MetricUpdate<UpdateT>> extractCumulatives(
+      MetricsMap<MetricName, CellT> cells) {
+    ImmutableList.Builder<MetricUpdate<UpdateT>> updates = ImmutableList.builder();
+    for (Map.Entry<MetricName, CellT> cell : cells.entries()) {
+      UpdateT update = checkNotNull(cell.getValue().getCumulative());
+      updates.add(MetricUpdate.create(MetricKey.create(stepName, cell.getKey()), update));
+    }
+    return updates.build();
+  }
+
+  /**
+   * Return the {@link MetricUpdates} representing the cumulative values of all metrics in this
+   * container.
+   */
+  public MetricUpdates getCumulative() {
+    return MetricUpdates.create(
+        extractCumulatives(counters),
+        extractCumulatives(distributions),
+        extractCumulatives(gauges));
+  }
+
+  /**
+   * Update values of this {@link MetricsContainerImpl} by merging the value of another cell.
+   */
+  public void update(MetricsContainerImpl other) {
+    updateCounters(counters, other.counters);
+    updateDistributions(distributions, other.distributions);
+    updateGauges(gauges, other.gauges);
+  }
+
+  private void updateCounters(
+      MetricsMap<MetricName, CounterCell> current,
+      MetricsMap<MetricName, CounterCell> updates) {
+    for (Map.Entry<MetricName, CounterCell> counter : updates.entries()) {
+      current.get(counter.getKey()).inc(counter.getValue().getCumulative());
+    }
+  }
+
+  private void updateDistributions(
+      MetricsMap<MetricName, DistributionCell> current,
+      MetricsMap<MetricName, DistributionCell> updates) {
+    for (Map.Entry<MetricName, DistributionCell> counter : updates.entries()) {
+      current.get(counter.getKey()).update(counter.getValue().getCumulative());
+    }
+  }
+
+  private void updateGauges(
+      MetricsMap<MetricName, GaugeCell> current,
+      MetricsMap<MetricName, GaugeCell> updates) {
+    for (Map.Entry<MetricName, GaugeCell> counter : updates.entries()) {
+      current.get(counter.getKey()).update(counter.getValue().getCumulative());
+    }
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainerStepMap.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java
similarity index 93%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainerStepMap.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java
index d01e970f7283..68238e49c80f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainerStepMap.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java
@@ -15,7 +15,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import com.google.common.base.Function;
 import com.google.common.base.Predicate;
@@ -24,7 +25,14 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
+import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
+import org.apache.beam.sdk.metrics.DistributionResult;
+import org.apache.beam.sdk.metrics.GaugeResult;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.apache.beam.sdk.metrics.MetricQueryResults;
+import org.apache.beam.sdk.metrics.MetricResult;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.metrics.MetricsFilter;
 
 /**
  * Metrics containers by step.
@@ -32,7 +40,7 @@
  * <p>This class is not thread-safe.</p>
  */
 public class MetricsContainerStepMap implements Serializable {
-  private Map<String, MetricsContainer> metricsContainers;
+  private Map<String, MetricsContainerImpl> metricsContainers;
 
   public MetricsContainerStepMap() {
     this.metricsContainers = new ConcurrentHashMap<>();
@@ -41,9 +49,9 @@ public MetricsContainerStepMap() {
   /**
    * Returns the container for the given step name.
    */
-  public MetricsContainer getContainer(String stepName) {
+  public MetricsContainerImpl getContainer(String stepName) {
     if (!metricsContainers.containsKey(stepName)) {
-      metricsContainers.put(stepName, new MetricsContainer(stepName));
+      metricsContainers.put(stepName, new MetricsContainerImpl(stepName));
     }
     return metricsContainers.get(stepName);
   }
@@ -53,16 +61,16 @@ public MetricsContainer getContainer(String stepName) {
    * {@link MetricsContainerStepMap}.
    */
   public void updateAll(MetricsContainerStepMap other) {
-    for (Map.Entry<String, MetricsContainer> container : other.metricsContainers.entrySet()) {
+    for (Map.Entry<String, MetricsContainerImpl> container : other.metricsContainers.entrySet()) {
       getContainer(container.getKey()).update(container.getValue());
     }
   }
 
   /**
-   * Update {@link MetricsContainer} for given step in this map with all values from given
-   * {@link MetricsContainer}.
+   * Update {@link MetricsContainerImpl} for given step in this map with all values from given
+   * {@link MetricsContainerImpl}.
    */
-  public void update(String step, MetricsContainer container) {
+  public void update(String step, MetricsContainerImpl container) {
     getContainer(step).update(container);
   }
 
@@ -94,7 +102,7 @@ public static MetricResults asAttemptedOnlyMetricResults(
     return new MetricsContainerStepMapMetricResults(attemptedMetricsContainers);
   }
 
-  private Map<String, MetricsContainer> getMetricsContainers() {
+  private Map<String, MetricsContainerImpl> getMetricsContainers() {
     return metricsContainers;
   }
 
@@ -120,7 +128,7 @@ private MetricsContainerStepMapMetricResults(
         MetricsContainerStepMap attemptedMetricsContainers,
         MetricsContainerStepMap committedMetricsContainers,
         boolean isCommittedSupported) {
-      for (MetricsContainer container
+      for (MetricsContainerImpl container
           : attemptedMetricsContainers.getMetricsContainers().values()) {
         MetricUpdates cumulative = container.getCumulative();
         mergeCounters(counters, cumulative.counterUpdates(), attemptedCounterUpdateFn());
@@ -128,7 +136,7 @@ private MetricsContainerStepMapMetricResults(
             attemptedDistributionUpdateFn());
         mergeGauges(gauges, cumulative.gaugeUpdates(), attemptedGaugeUpdateFn());
       }
-      for (MetricsContainer container
+      for (MetricsContainerImpl container
           : committedMetricsContainers.getMetricsContainers().values()) {
         MetricUpdates cumulative = container.getCumulative();
         mergeCounters(counters, cumulative.counterUpdates(), committedCounterUpdateFn());
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsMap.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsMap.java
similarity index 98%
rename from sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsMap.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsMap.java
index 8c26f1869755..9f080760148d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsMap.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsMap.java
@@ -15,7 +15,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.Iterables;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
new file mode 100644
index 000000000000..263a705ac033
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for runners to implement metrics.
+ */
+package org.apache.beam.runners.core.metrics;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java
index 88c6ab6e7e1b..9ee79d74ee6d 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java
@@ -26,8 +26,8 @@
 import com.google.common.collect.Iterables;
 import java.util.Arrays;
 import org.apache.beam.runners.core.LateDataDroppingDoFnRunner.LateDataFilter;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
@@ -58,7 +58,8 @@ public void setUp() {
 
   @Test
   public void testLateDataFilter() throws Exception {
-    MetricsEnvironment.setCurrentContainer(new MetricsContainer("any"));
+    MetricsContainerImpl container = new MetricsContainerImpl("any");
+    MetricsEnvironment.setCurrentContainer(container);
     when(mockTimerInternals.currentInputWatermarkTime()).thenReturn(new Instant(15L));
 
     LateDataFilter lateDataFilter = new LateDataFilter(
@@ -77,14 +78,14 @@ public void testLateDataFilter() throws Exception {
         createDatum(16, 16L),
         createDatum(18, 18L));
     assertThat(expected, containsInAnyOrder(Iterables.toArray(actual, WindowedValue.class)));
-    long droppedValues = MetricsEnvironment.getCurrentContainer().getCounter(
+    long droppedValues = container.getCounter(
         MetricName.named(LateDataDroppingDoFnRunner.class,
             LateDataDroppingDoFnRunner.DROPPED_DUE_TO_LATENESS))
         .getCumulative().longValue();
     assertEquals(1, droppedValues);
     // Ensure that reiterating returns the same results and doesn't increment the counter again.
     assertThat(expected, containsInAnyOrder(Iterables.toArray(actual, WindowedValue.class)));
-    droppedValues = MetricsEnvironment.getCurrentContainer().getCounter(
+    droppedValues = container.getCounter(
         MetricName.named(LateDataDroppingDoFnRunner.class,
             LateDataDroppingDoFnRunner.DROPPED_DUE_TO_LATENESS))
         .getCumulative().longValue();
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
index 3dd98e0b7cf7..9e71300fb335 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java
@@ -38,10 +38,10 @@
 
 import com.google.common.collect.Iterables;
 import java.util.List;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.runners.core.triggers.TriggerStateMachine;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -145,7 +145,8 @@ public Void answer(InvocationOnMock invocation) throws Exception {
   @Test
   public void testOnElementBufferingDiscarding() throws Exception {
     // Test basic execution of a trigger using a non-combining window set and discarding mode.
-    MetricsEnvironment.setCurrentContainer(new MetricsContainer("any"));
+    MetricsContainerImpl container = new MetricsContainerImpl("any");
+    MetricsEnvironment.setCurrentContainer(container);
     ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
         ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine,
             AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100),
@@ -170,8 +171,7 @@ public void testOnElementBufferingDiscarding() throws Exception {
     // This element shouldn't be seen, because the trigger has finished
     injectElement(tester, 4);
 
-    long droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
-        MetricName.named(ReduceFnRunner.class,
+    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
     assertEquals(1, droppedElements);
@@ -423,7 +423,8 @@ public Integer answer(InvocationOnMock invocation) throws Throwable {
 
   @Test
   public void testWatermarkHoldAndLateData() throws Exception {
-    MetricsEnvironment.setCurrentContainer(new MetricsContainer("any"));
+    MetricsContainerImpl container = new MetricsContainerImpl("any");
+    MetricsEnvironment.setCurrentContainer(container);
     // Test handling of late data. Specifically, ensure the watermark hold is correct.
     ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
         ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine,
@@ -452,7 +453,7 @@ public void testWatermarkHoldAndLateData() throws Exception {
     // Holding for the end-of-window transition.
     assertEquals(new Instant(9), tester.getWatermarkHold());
     // Nothing dropped.
-    long droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
+    long droppedElements = container.getCounter(
         MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
@@ -514,7 +515,7 @@ public void testWatermarkHoldAndLateData() throws Exception {
     // Because we're about to expire the window, we output it.
     when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
     injectElement(tester, 8);
-    droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
+    droppedElements = container.getCounter(
         MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
@@ -1083,7 +1084,8 @@ public void testMergingWithClosedDoesNotPoison() throws Exception {
    */
   @Test
   public void testDropDataMultipleWindowsFinishedTrigger() throws Exception {
-    MetricsEnvironment.setCurrentContainer(new MetricsContainer("any"));
+    MetricsContainerImpl container = new MetricsContainerImpl("any");
+    MetricsEnvironment.setCurrentContainer(container);
     ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(
         WindowingStrategy.of(
             SlidingWindows.of(Duration.millis(100)).every(Duration.millis(30)))
@@ -1097,7 +1099,7 @@ public void testDropDataMultipleWindowsFinishedTrigger() throws Exception {
         // assigned to [-30, 70), [0, 100), [30, 130)
         TimestampedValue.of(12, new Instant(40)));
 
-    long droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
+    long droppedElements = container.getCounter(
         MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
@@ -1109,7 +1111,7 @@ public void testDropDataMultipleWindowsFinishedTrigger() throws Exception {
         // but [-30, 70) is closed by the trigger
         TimestampedValue.of(14, new Instant(60)));
 
-    droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
+    droppedElements = container.getCounter(
         MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
@@ -1120,7 +1122,7 @@ public void testDropDataMultipleWindowsFinishedTrigger() throws Exception {
     // but they are all closed
     tester.injectElements(TimestampedValue.of(16, new Instant(40)));
 
-    droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
+    droppedElements = container.getCounter(
         MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
@@ -1129,7 +1131,8 @@ public void testDropDataMultipleWindowsFinishedTrigger() throws Exception {
 
   @Test
   public void testIdempotentEmptyPanesDiscarding() throws Exception {
-    MetricsEnvironment.setCurrentContainer(new MetricsContainer("any"));
+    MetricsContainerImpl container = new MetricsContainerImpl("any");
+    MetricsEnvironment.setCurrentContainer(container);
     // Test uninteresting (empty) panes don't increment the index or otherwise
     // modify PaneInfo.
     ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
@@ -1170,7 +1173,7 @@ public void testIdempotentEmptyPanesDiscarding() throws Exception {
     assertTrue(tester.isMarkedFinished(firstWindow));
     tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
 
-    long droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
+    long droppedElements = container.getCounter(
         MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
@@ -1179,7 +1182,8 @@ public void testIdempotentEmptyPanesDiscarding() throws Exception {
 
   @Test
   public void testIdempotentEmptyPanesAccumulating() throws Exception {
-    MetricsEnvironment.setCurrentContainer(new MetricsContainer("any"));
+    MetricsContainerImpl container = new MetricsContainerImpl("any");
+    MetricsEnvironment.setCurrentContainer(container);
     // Test uninteresting (empty) panes don't increment the index or otherwise
     // modify PaneInfo.
     ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
@@ -1222,7 +1226,7 @@ public void testIdempotentEmptyPanesAccumulating() throws Exception {
     assertTrue(tester.isMarkedFinished(firstWindow));
     tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
 
-    long droppedElements = MetricsEnvironment.getCurrentContainer().getCounter(
+    long droppedElements = container.getCounter(
         MetricName.named(ReduceFnRunner.class,
             ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW))
         .getCumulative().longValue();
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index 76351e40cad3..5172f433135a 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -25,10 +25,10 @@
 import com.google.common.base.MoreObjects;
 import java.util.Collections;
 import org.apache.beam.runners.core.BaseExecutionContext.StepContext;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.state.StateSpec;
 import org.apache.beam.sdk.state.StateSpecs;
@@ -93,7 +93,8 @@ public void setup() {
 
   @Test
   public void testLateDropping() throws Exception {
-    MetricsEnvironment.setCurrentContainer(new MetricsContainer("any"));
+    MetricsContainerImpl container = new MetricsContainerImpl("any");
+    MetricsEnvironment.setCurrentContainer(container);
 
     timerInternals.advanceInputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE));
     timerInternals.advanceOutputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE));
@@ -117,9 +118,8 @@ public void testLateDropping() throws Exception {
         WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING));
 
 
-    long droppedValues = MetricsEnvironment.getCurrentContainer().getCounter(
-        MetricName.named(StatefulDoFnRunner.class,
-            StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)).getCumulative().longValue();
+    long droppedValues = container.getCounter(MetricName.named(StatefulDoFnRunner.class,
+        StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)).getCumulative().longValue();
     assertEquals(1L, droppedValues);
 
     runner.finishBundle();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/CounterCellTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/CounterCellTest.java
similarity index 88%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/CounterCellTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/CounterCellTest.java
index 26554d4bc9a4..12906d988a46 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/CounterCellTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/CounterCellTest.java
@@ -16,11 +16,12 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertThat;
 
+import org.apache.beam.sdk.metrics.MetricName;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -31,12 +32,12 @@
 @RunWith(JUnit4.class)
 public class CounterCellTest {
 
-  private CounterCell cell = new CounterCell();
+  private CounterCell cell = new CounterCell(MetricName.named("hello", "world"));
 
   @Test
   public void testDeltaAndCumulative() {
-    cell.update(5);
-    cell.update(7);
+    cell.inc(5);
+    cell.inc(7);
     assertThat(cell.getCumulative(), equalTo(12L));
     assertThat("getCumulative is idempotent", cell.getCumulative(), equalTo(12L));
 
@@ -45,7 +46,7 @@ public void testDeltaAndCumulative() {
     assertThat(cell.getDirty().beforeCommit(), equalTo(false));
     assertThat(cell.getCumulative(), equalTo(12L));
 
-    cell.update(30);
+    cell.inc(30);
     assertThat(cell.getCumulative(), equalTo(42L));
 
     assertThat(cell.getDirty().beforeCommit(), equalTo(true));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/DirtyStateTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/DirtyStateTest.java
similarity index 97%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/DirtyStateTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/DirtyStateTest.java
index d00f8cd8c435..e5c889895575 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/DirtyStateTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/DirtyStateTest.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/DistributionCellTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/DistributionCellTest.java
similarity index 90%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/DistributionCellTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/DistributionCellTest.java
index 07e0b266e0d9..c0b045aa603f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/DistributionCellTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/DistributionCellTest.java
@@ -16,11 +16,12 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertThat;
 
+import org.apache.beam.sdk.metrics.MetricName;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -30,7 +31,7 @@
  */
 @RunWith(JUnit4.class)
 public class DistributionCellTest {
-  private DistributionCell cell = new DistributionCell();
+  private DistributionCell cell = new DistributionCell(MetricName.named("hello", "world"));
 
   @Test
   public void testDeltaAndCumulative() {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/GaugeCellTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/GaugeCellTest.java
similarity index 90%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/GaugeCellTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/GaugeCellTest.java
index 7dbfc4ba822b..cfaf92d5bd04 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/GaugeCellTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/GaugeCellTest.java
@@ -15,18 +15,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
+
+package org.apache.beam.runners.core.metrics;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertThat;
 
+import org.apache.beam.sdk.metrics.MetricName;
 import org.junit.Test;
 
 /**
  * Tests for {@link GaugeCell}.
  */
 public class GaugeCellTest {
-  private GaugeCell cell = new GaugeCell();
+  private GaugeCell cell = new GaugeCell(MetricName.named("hello", "world"));
 
   @Test
   public void testDeltaAndCumulative() {
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricUpdateMatchers.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricUpdateMatchers.java
new file mode 100644
index 000000000000..a5ee874feb98
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricUpdateMatchers.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.metrics;
+
+import java.util.Objects;
+import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeMatcher;
+
+/**
+ * Matchers for {@link MetricUpdate}.
+ */
+public class MetricUpdateMatchers {
+
+  /**
+   * Matches a {@link MetricUpdate} with the given name and contents.
+   *
+   * <p>Visible since it may be used in runner-specific tests.
+   */
+  public static <T> Matcher<MetricUpdate<T>> metricUpdate(final String name, final T update) {
+    return new TypeSafeMatcher<MetricUpdate<T>>() {
+      @Override
+      protected boolean matchesSafely(MetricUpdate<T> item) {
+        return Objects.equals(name, item.getKey().metricName().name())
+            && Objects.equals(update, item.getUpdate());
+      }
+
+      @Override
+      public void describeTo(Description description) {
+        description
+            .appendText("MetricUpdate{name=").appendValue(name)
+            .appendText(", update=").appendValue(update)
+            .appendText("}");
+      }
+    };
+  }
+
+  /**
+   * Matches a {@link MetricUpdate} with the given namespace, name, step and contents.
+   *
+   * <p>Visible since it may be used in runner-specific tests.
+   */
+  public static <T> Matcher<MetricUpdate<T>> metricUpdate(
+      final String namespace, final String name, final String step, final T update) {
+    return new TypeSafeMatcher<MetricUpdate<T>>() {
+      @Override
+      protected boolean matchesSafely(MetricUpdate<T> item) {
+        return Objects.equals(namespace, item.getKey().metricName().namespace())
+            && Objects.equals(name, item.getKey().metricName().name())
+            && Objects.equals(step, item.getKey().stepName())
+            && Objects.equals(update, item.getUpdate());
+      }
+
+      @Override
+      public void describeTo(Description description) {
+        description
+            .appendText("MetricUpdate{inNamespace=").appendValue(namespace)
+            .appendText(", name=").appendValue(name)
+            .appendText(", step=").appendValue(step)
+            .appendText(", update=").appendValue(update)
+            .appendText("}");
+      }
+    };
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java
similarity index 88%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java
index 38c00d30e605..b304d3bb6f6b 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerImplTest.java
@@ -16,27 +16,28 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.metricUpdate;
+import static org.apache.beam.runners.core.metrics.MetricUpdateMatchers.metricUpdate;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
 import static org.junit.Assert.assertThat;
 
+import org.apache.beam.sdk.metrics.MetricName;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /**
- * Tests for {@link MetricsContainer}.
+ * Tests for {@link MetricsContainerImpl}.
  */
 @RunWith(JUnit4.class)
-public class MetricsContainerTest {
+public class MetricsContainerImplTest {
 
   @Test
   public void testCounterDeltas() {
-    MetricsContainer container = new MetricsContainer("step1");
+    MetricsContainerImpl container = new MetricsContainerImpl("step1");
     CounterCell c1 = container.getCounter(MetricName.named("ns", "name1"));
     CounterCell c2 = container.getCounter(MetricName.named("ns", "name2"));
     assertThat("All counters should start out dirty",
@@ -47,8 +48,8 @@ public void testCounterDeltas() {
     assertThat("After commit no counters should be dirty",
         container.getUpdates().counterUpdates(), emptyIterable());
 
-    c1.update(5L);
-    c2.update(4L);
+    c1.inc(5L);
+    c2.inc(4L);
 
     assertThat(container.getUpdates().counterUpdates(), containsInAnyOrder(
         metricUpdate("name1", 5L),
@@ -63,19 +64,19 @@ public void testCounterDeltas() {
     assertThat("After commit there are no updates",
         container.getUpdates().counterUpdates(), emptyIterable());
 
-    c1.update(8L);
+    c1.inc(8L);
     assertThat(container.getUpdates().counterUpdates(), contains(
         metricUpdate("name1", 13L)));
   }
 
   @Test
   public void testCounterCumulatives() {
-    MetricsContainer container = new MetricsContainer("step1");
+    MetricsContainerImpl container = new MetricsContainerImpl("step1");
     CounterCell c1 = container.getCounter(MetricName.named("ns", "name1"));
     CounterCell c2 = container.getCounter(MetricName.named("ns", "name2"));
-    c1.update(2L);
-    c2.update(4L);
-    c1.update(3L);
+    c1.inc(2L);
+    c2.inc(4L);
+    c1.inc(3L);
 
     container.getUpdates();
     container.commitUpdates();
@@ -84,7 +85,7 @@ public void testCounterCumulatives() {
         metricUpdate("name1", 5L),
         metricUpdate("name2", 4L)));
 
-    c1.update(8L);
+    c1.inc(8L);
     assertThat(container.getCumulative().counterUpdates(), containsInAnyOrder(
         metricUpdate("name1", 13L),
         metricUpdate("name2", 4L)));
@@ -92,7 +93,7 @@ public void testCounterCumulatives() {
 
   @Test
   public void testDistributionDeltas() {
-    MetricsContainer container = new MetricsContainer("step1");
+    MetricsContainerImpl container = new MetricsContainerImpl("step1");
     DistributionCell c1 = container.getDistribution(MetricName.named("ns", "name1"));
     DistributionCell c2 = container.getDistribution(MetricName.named("ns", "name2"));
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerStepMapTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMapTest.java
similarity index 66%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerStepMapTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMapTest.java
index 0428ce1aa84e..3074cc02326d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsContainerStepMapTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMapTest.java
@@ -15,16 +15,27 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.metricsResult;
-import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
-import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asMetricResults;
+package org.apache.beam.runners.core.metrics;
+
+import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asMetricResults;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.metricsResult;
 import static org.hamcrest.Matchers.hasItem;
 import static org.junit.Assert.assertThat;
 
 import java.io.Closeable;
 import java.io.IOException;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Distribution;
+import org.apache.beam.sdk.metrics.DistributionResult;
+import org.apache.beam.sdk.metrics.Gauge;
+import org.apache.beam.sdk.metrics.GaugeResult;
+import org.apache.beam.sdk.metrics.MetricQueryResults;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.metrics.MetricsEnvironment;
+import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.hamcrest.collection.IsIterableWithSize;
 import org.joda.time.Instant;
 import org.junit.Rule;
@@ -39,26 +50,23 @@ public class MetricsContainerStepMapTest {
   private static final String NAMESPACE = MetricsContainerStepMapTest.class.getName();
   private static final String STEP1 = "myStep1";
   private static final String STEP2 = "myStep2";
+  private static final String COUNTER_NAME = "myCounter";
+  private static final String DISTRIBUTION_NAME = "myDistribution";
+  private static final String GAUGE_NAME = "myGauge";
 
   private static final long VALUE = 100;
 
   private static final Counter counter =
-      Metrics.counter(
-          MetricsContainerStepMapTest.class,
-          "myCounter");
+      Metrics.counter(MetricsContainerStepMapTest.class, COUNTER_NAME);
   private static final Distribution distribution =
-      Metrics.distribution(
-          MetricsContainerStepMapTest.class,
-          "myDistribution");
+      Metrics.distribution(MetricsContainerStepMapTest.class, DISTRIBUTION_NAME);
   private static final Gauge gauge =
-      Metrics.gauge(
-          MetricsContainerStepMapTest.class,
-          "myGauge");
+      Metrics.gauge(MetricsContainerStepMapTest.class, GAUGE_NAME);
 
-  private static final MetricsContainer metricsContainer;
+  private static final MetricsContainerImpl metricsContainer;
 
   static {
-    metricsContainer = new MetricsContainer(null);
+    metricsContainer = new MetricsContainerImpl(null);
     try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
       counter.inc(VALUE);
       distribution.update(VALUE);
@@ -89,10 +97,11 @@ public void testAttemptedAccumulatedMetricResults() {
     assertIterableSize(step1res.distributions(), 1);
     assertIterableSize(step1res.gauges(), 1);
 
-    assertCounter(step1res, STEP1, VALUE, false);
-    assertDistribution(step1res, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
+    assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, false);
+    assertDistribution(DISTRIBUTION_NAME,
+        step1res, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
         false);
-    assertGauge(step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);
+    assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);
 
     MetricQueryResults step2res =
         metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build());
@@ -101,10 +110,12 @@ public void testAttemptedAccumulatedMetricResults() {
     assertIterableSize(step2res.distributions(), 1);
     assertIterableSize(step2res.gauges(), 1);
 
-    assertCounter(step2res, STEP2, VALUE * 2, false);
-    assertDistribution(step2res, STEP2, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
-        false);
-    assertGauge(step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);
+    assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 2, false);
+    assertDistribution(
+        DISTRIBUTION_NAME, step2res, STEP2,
+        DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2), false);
+    assertGauge(
+        GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);
 
     MetricQueryResults allres =
         metricResults.queryMetrics(MetricsFilter.builder().build());
@@ -127,7 +138,7 @@ public void testCounterCommittedUnsupportedInAttemptedAccumulatedMetricResults()
     thrown.expect(UnsupportedOperationException.class);
     thrown.expectMessage("This runner does not currently support committed metrics results.");
 
-    assertCounter(step1res, STEP1, VALUE, true);
+    assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, true);
   }
 
   @Test
@@ -143,7 +154,7 @@ public void testDistributionCommittedUnsupportedInAttemptedAccumulatedMetricResu
     thrown.expect(UnsupportedOperationException.class);
     thrown.expectMessage("This runner does not currently support committed metrics results.");
 
-    assertDistribution(step1res, STEP1, DistributionResult.ZERO, true);
+    assertDistribution(DISTRIBUTION_NAME, step1res, STEP1, DistributionResult.ZERO, true);
   }
 
   @Test
@@ -159,7 +170,7 @@ public void testGaugeCommittedUnsupportedInAttemptedAccumulatedMetricResults() {
     thrown.expect(UnsupportedOperationException.class);
     thrown.expectMessage("This runner does not currently support committed metrics results.");
 
-    assertGauge(step1res, STEP1, GaugeResult.empty(), true);
+    assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.empty(), true);
   }
 
   @Test
@@ -186,15 +197,16 @@ public void testAttemptedAndCommittedAccumulatedMetricResults() {
     assertIterableSize(step1res.distributions(), 1);
     assertIterableSize(step1res.gauges(), 1);
 
-    assertCounter(step1res, STEP1, VALUE * 2, false);
-    assertDistribution(step1res, STEP1, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
-        false);
-    assertGauge(step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);
+    assertCounter(COUNTER_NAME, step1res, STEP1, VALUE * 2, false);
+    assertDistribution(
+        DISTRIBUTION_NAME, step1res, STEP1,
+        DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2), false);
+    assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false);
 
-    assertCounter(step1res, STEP1, VALUE, true);
-    assertDistribution(step1res, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2),
-        true);
-    assertGauge(step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), true);
+    assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, true);
+    assertDistribution(DISTRIBUTION_NAME, step1res, STEP1,
+        DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2), true);
+    assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), true);
 
     MetricQueryResults step2res =
         metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build());
@@ -203,15 +215,15 @@ public void testAttemptedAndCommittedAccumulatedMetricResults() {
     assertIterableSize(step2res.distributions(), 1);
     assertIterableSize(step2res.gauges(), 1);
 
-    assertCounter(step2res, STEP2, VALUE * 3, false);
-    assertDistribution(step2res, STEP2, DistributionResult.create(VALUE * 9, 6, VALUE, VALUE * 2),
-        false);
-    assertGauge(step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);
+    assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 3, false);
+    assertDistribution(DISTRIBUTION_NAME, step2res, STEP2,
+        DistributionResult.create(VALUE * 9, 6, VALUE, VALUE * 2), false);
+    assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false);
 
-    assertCounter(step2res, STEP2, VALUE * 2, true);
-    assertDistribution(step2res, STEP2, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2),
-        true);
-    assertGauge(step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), true);
+    assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 2, true);
+    assertDistribution(DISTRIBUTION_NAME, step2res, STEP2,
+        DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2), true);
+    assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), true);
 
     MetricQueryResults allres =
         metricResults.queryMetrics(MetricsFilter.builder().build());
@@ -226,33 +238,35 @@ private <T> void assertIterableSize(Iterable<T> iterable, int size) {
   }
 
   private void assertCounter(
+      String name,
       MetricQueryResults metricQueryResults,
       String step,
       Long expected,
       boolean isCommitted) {
     assertThat(
         metricQueryResults.counters(),
-        hasItem(metricsResult(NAMESPACE, counter.getName().name(), step, expected, isCommitted)));
+        hasItem(metricsResult(NAMESPACE, name, step, expected, isCommitted)));
   }
 
   private void assertDistribution(
+      String name,
       MetricQueryResults metricQueryResults,
       String step,
       DistributionResult expected,
       boolean isCommitted) {
     assertThat(
         metricQueryResults.distributions(),
-        hasItem(metricsResult(NAMESPACE, distribution.getName().name(), step, expected,
-            isCommitted)));
+        hasItem(metricsResult(NAMESPACE, name, step, expected, isCommitted)));
   }
 
   private void assertGauge(
+      String name,
       MetricQueryResults metricQueryResults,
       String step,
       GaugeResult expected,
       boolean isCommitted) {
     assertThat(
         metricQueryResults.gauges(),
-        hasItem(metricsResult(NAMESPACE, gauge.getName().name(), step, expected, isCommitted)));
+        hasItem(metricsResult(NAMESPACE, name, step, expected, isCommitted)));
   }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsMapTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsMapTest.java
similarity index 98%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsMapTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsMapTest.java
index 4104f8ddab62..96210ddb5e60 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsMapTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MetricsMapTest.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.beam.sdk.metrics;
+package org.apache.beam.runners.core.metrics;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.not;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
index b7cd6e73f07a..8169332b8173 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java
@@ -32,20 +32,20 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.concurrent.GuardedBy;
-import org.apache.beam.sdk.metrics.DistributionData;
+import org.apache.beam.runners.core.metrics.DistributionData;
+import org.apache.beam.runners.core.metrics.GaugeData;
+import org.apache.beam.runners.core.metrics.MetricFiltering;
+import org.apache.beam.runners.core.metrics.MetricKey;
+import org.apache.beam.runners.core.metrics.MetricUpdates;
+import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
+import org.apache.beam.runners.core.metrics.MetricsMap;
 import org.apache.beam.sdk.metrics.DistributionResult;
-import org.apache.beam.sdk.metrics.GaugeData;
 import org.apache.beam.sdk.metrics.GaugeResult;
-import org.apache.beam.sdk.metrics.MetricFiltering;
-import org.apache.beam.sdk.metrics.MetricKey;
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
 import org.apache.beam.sdk.metrics.MetricResult;
 import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.metrics.MetricUpdates;
-import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
 import org.apache.beam.sdk.metrics.MetricsFilter;
-import org.apache.beam.sdk.metrics.MetricsMap;
 
 /**
  * Implementation of {@link MetricResults} for the Direct Runner.
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
index c7f78472f627..6efb99fa1aec 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
@@ -23,9 +23,9 @@
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.Set;
+import org.apache.beam.runners.core.metrics.MetricUpdates;
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
-import org.apache.beam.sdk.metrics.MetricUpdates;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
index 56f8650a1034..76d817b960c0 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
@@ -21,8 +21,8 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.concurrent.Callable;
-import org.apache.beam.sdk.metrics.MetricUpdates;
-import org.apache.beam.sdk.metrics.MetricsContainer;
+import org.apache.beam.runners.core.metrics.MetricUpdates;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -92,7 +92,7 @@ private TransformExecutor(
 
   @Override
   public void run() {
-    MetricsContainer metricsContainer = new MetricsContainer(transform.getFullName());
+    MetricsContainerImpl metricsContainer = new MetricsContainerImpl(transform.getFullName());
     try (Closeable metricsScope = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) {
       Collection<ModelEnforcement<T>> enforcements = new ArrayList<>();
       for (ModelEnforcementFactory enforcementFactory : modelEnforcements) {
@@ -134,7 +134,7 @@ public void run() {
    */
   private void processElements(
       TransformEvaluator<T> evaluator,
-      MetricsContainer metricsContainer,
+      MetricsContainerImpl metricsContainer,
       Collection<ModelEnforcement<T>> enforcements)
       throws Exception {
     if (inputBundle != null) {
@@ -167,7 +167,7 @@ private void processElements(
    *         {@link TransformEvaluator#finishBundle()}
    */
   private TransformResult<T> finishBundle(
-      TransformEvaluator<T> evaluator, MetricsContainer metricsContainer,
+      TransformEvaluator<T> evaluator, MetricsContainerImpl metricsContainer,
       Collection<ModelEnforcement<T>> enforcements)
       throws Exception {
     TransformResult<T> result =
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
index 8d7aeda72fcf..554c2d65aa42 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
@@ -19,9 +19,9 @@
 
 import java.util.Set;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.metrics.MetricUpdates;
 import org.apache.beam.runners.direct.CommittedResult.OutputType;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
-import org.apache.beam.sdk.metrics.MetricUpdates;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java
index d5d0affa0ef3..8cdf3239db4a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java
@@ -18,23 +18,23 @@
 
 package org.apache.beam.runners.direct;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.attemptedMetricsResult;
-import static org.apache.beam.sdk.metrics.MetricMatchers.committedMetricsResult;
 import static org.apache.beam.sdk.metrics.MetricNameFilter.inNamespace;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.attemptedMetricsResult;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.committedMetricsResult;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.junit.Assert.assertThat;
 
 import com.google.common.collect.ImmutableList;
-import org.apache.beam.sdk.metrics.DistributionData;
+import org.apache.beam.runners.core.metrics.DistributionData;
+import org.apache.beam.runners.core.metrics.GaugeData;
+import org.apache.beam.runners.core.metrics.MetricKey;
+import org.apache.beam.runners.core.metrics.MetricUpdates;
+import org.apache.beam.runners.core.metrics.MetricUpdates.MetricUpdate;
 import org.apache.beam.sdk.metrics.DistributionResult;
-import org.apache.beam.sdk.metrics.GaugeData;
 import org.apache.beam.sdk.metrics.GaugeResult;
-import org.apache.beam.sdk.metrics.MetricKey;
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
-import org.apache.beam.sdk.metrics.MetricUpdates;
-import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
 import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.joda.time.Instant;
 import org.junit.Before;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
index 038895ad267d..b1f956db2716 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
@@ -17,15 +17,15 @@
  */
 package org.apache.beam.runners.flink;
 
-import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.flink.metrics.FlinkMetricContainer;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.joda.time.Duration;
 
 /**
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
index 40191d26b0a2..ccce127eeed9 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/DoFnRunnerWithMetricsUpdate.java
@@ -20,6 +20,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import org.apache.beam.runners.core.DoFnRunner;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -29,7 +30,7 @@
 
 /**
  * {@link DoFnRunner} decorator which registers
- * {@link org.apache.beam.sdk.metrics.MetricsContainer}. It updates metrics to Flink metrics and
+ * {@link MetricsContainerImpl}. It updates metrics to Flink metrics and
  * accumulators in {@link #finishBundle()}.
  */
 public class DoFnRunnerWithMetricsUpdate<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java
index f81205e9b8cc..677051e2bfac 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java
@@ -17,17 +17,18 @@
  */
 package org.apache.beam.runners.flink.metrics;
 
-import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
 
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.DistributionResult;
 import org.apache.beam.sdk.metrics.GaugeResult;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
 import org.apache.beam.sdk.metrics.MetricResult;
 import org.apache.beam.sdk.metrics.MetricResults;
 import org.apache.beam.sdk.metrics.MetricsContainer;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsFilter;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.functions.RuntimeContext;
@@ -37,7 +38,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Helper class for holding a {@link MetricsContainer} and forwarding Beam metrics to
+ * Helper class for holding a {@link MetricsContainerImpl} and forwarding Beam metrics to
  * Flink accumulators and metrics.
  */
 public class FlinkMetricContainer {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java
index a9dc2ce92be4..d8b5d2516d1b 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/MetricsAccumulator.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.runners.flink.metrics;
 
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.accumulators.SimpleAccumulator;
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java
index 64738cce2af8..85d6426218db 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/ReaderInvocationUtil.java
@@ -19,6 +19,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.runners.flink.FlinkPipelineOptions;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
@@ -26,8 +27,8 @@
 
 /**
  * Util for invoking {@link Source.Reader} methods that might require a
- * {@link org.apache.beam.sdk.metrics.MetricsContainer} to be active.
- * Source.Reader decorator which registers {@link org.apache.beam.sdk.metrics.MetricsContainer}.
+ * {@link MetricsContainerImpl} to be active.
+ * Source.Reader decorator which registers {@link MetricsContainerImpl}.
  * It update metrics to Flink metric and accumulator in start and advance.
  */
 public class ReaderInvocationUtil<OutputT, ReaderT extends Source.Reader<OutputT>> {
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 380933576cdc..f67997652853 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170508-pr2933</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170510</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
index aa80959932e2..a6a6a43e3f65 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
@@ -28,10 +28,10 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import org.apache.beam.runners.core.metrics.MetricFiltering;
+import org.apache.beam.runners.core.metrics.MetricKey;
 import org.apache.beam.sdk.metrics.DistributionResult;
 import org.apache.beam.sdk.metrics.GaugeResult;
-import org.apache.beam.sdk.metrics.MetricFiltering;
-import org.apache.beam.sdk.metrics.MetricKey;
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
 import org.apache.beam.sdk.metrics.MetricResult;
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
index bca3428132e9..85a0979b3c69 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
@@ -17,8 +17,8 @@
  */
 package org.apache.beam.runners.dataflow;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.attemptedMetricsResult;
-import static org.apache.beam.sdk.metrics.MetricMatchers.committedMetricsResult;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.attemptedMetricsResult;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.committedMetricsResult;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
index 3986e337dd71..7fd6962f5a72 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
@@ -18,7 +18,7 @@
 
 package org.apache.beam.runners.spark;
 
-import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
 
 import java.io.IOException;
 import java.util.Objects;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
index aa89c59fc4b0..01cc1762413f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java
@@ -28,13 +28,13 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.metrics.MetricsContainer;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.spark.Accumulator;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
index 2a9de4b9846b..7106c73866a1 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Collections;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
@@ -36,7 +37,6 @@
 import org.apache.beam.sdk.metrics.Gauge;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.metrics.MetricsContainer;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
index 1dcfa2f11f0d..03b218769e54 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
@@ -21,10 +21,10 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import java.io.IOException;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.translation.streaming.Checkpoint;
 import org.apache.beam.runners.spark.translation.streaming.Checkpoint.CheckpointDir;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.spark.Accumulator;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java
index dee4ebc5f821..17cf71ff31fd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulatorParam.java
@@ -18,7 +18,7 @@
 
 package org.apache.beam.runners.spark.metrics;
 
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.spark.AccumulatorParam;
 
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java
index e4bd5989eb76..5e67445931ae 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java
@@ -18,19 +18,19 @@
 
 package org.apache.beam.runners.spark.metrics;
 
-import static org.apache.beam.sdk.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
+import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
 
 import com.codahale.metrics.Metric;
 import com.google.common.annotations.VisibleForTesting;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.DistributionResult;
 import org.apache.beam.sdk.metrics.GaugeResult;
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
 import org.apache.beam.sdk.metrics.MetricResult;
 import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsFilter;
 
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 37d9635ff5e7..1d843f90fb2f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -31,6 +31,8 @@
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.UnsupportedSideInputReader;
 import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.metrics.CounterCell;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.runners.spark.SparkPipelineOptions;
@@ -43,9 +45,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.metrics.CounterCell;
 import org.apache.beam.sdk.metrics.MetricName;
-import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -207,7 +207,7 @@ public JavaPairRDD<ByteArray, byte[]> call(
             new OutputWindowedValueHolder<>();
         // use in memory Aggregators since Spark Accumulators are not resilient
         // in stateful operators, once done with this partition.
-        final MetricsContainer cellProvider = new MetricsContainer("cellProvider");
+        final MetricsContainerImpl cellProvider = new MetricsContainerImpl("cellProvider");
         final CounterCell droppedDueToClosedWindow = cellProvider.getCounter(
             MetricName.named(SparkGroupAlsoByWindowViaWindowSet.class,
             GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER));
@@ -321,12 +321,12 @@ public JavaPairRDD<ByteArray, byte[]> call(
         long lateDropped = droppedDueToLateness.getCumulative();
         if (lateDropped > 0) {
           LOG.info(String.format("Dropped %d elements due to lateness.", lateDropped));
-          droppedDueToLateness.update(-droppedDueToLateness.getCumulative());
+          droppedDueToLateness.inc(-droppedDueToLateness.getCumulative());
         }
         long closedWindowDropped = droppedDueToClosedWindow.getCumulative();
         if (closedWindowDropped > 0) {
           LOG.info(String.format("Dropped %d elements due to closed window.", closedWindowDropped));
-          droppedDueToClosedWindow.update(-droppedDueToClosedWindow.getCumulative());
+          droppedDueToClosedWindow.inc(-droppedDueToClosedWindow.getCumulative());
         }
 
         return scala.collection.JavaConversions.asScalaIterator(outIter);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
index 17a3c7344bb0..549bd30a11aa 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
@@ -27,6 +27,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.runners.spark.io.EmptyCheckpointMark;
 import org.apache.beam.runners.spark.io.MicrobatchSource;
@@ -36,7 +37,6 @@
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.metrics.MetricsContainer;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
index 8349b09fab16..394b80b09317 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnRunnerWithMetrics.java
@@ -21,8 +21,9 @@
 import java.io.Closeable;
 import java.io.IOException;
 import org.apache.beam.runners.core.DoFnRunner;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsContainer;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -32,7 +33,7 @@
 
 
 /**
- * DoFnRunner decorator which registers {@link org.apache.beam.sdk.metrics.MetricsContainer}.
+ * DoFnRunner decorator which registers {@link MetricsContainerImpl}.
  */
 class DoFnRunnerWithMetrics<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
   private final DoFnRunner<InputT, OutputT> delegate;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
index ecf96b62f5a3..4a665417a078 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
@@ -27,10 +27,10 @@
 import java.util.Map;
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.spark.aggregators.NamedAggregators;
 import org.apache.beam.runners.spark.util.SideInputBroadcast;
 import org.apache.beam.runners.spark.util.SparkSideInputReader;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index acbac32840d6..b2ed3a9265d6 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -30,6 +30,7 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.beam.runners.core.SystemReduceFn;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
 import org.apache.beam.runners.spark.aggregators.NamedAggregators;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
@@ -41,7 +42,6 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.Create;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index f736e5305b3b..acb4a02f1e94 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -32,6 +32,7 @@
 import java.util.Queue;
 import java.util.concurrent.LinkedBlockingQueue;
 import javax.annotation.Nonnull;
+import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
 import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
 import org.apache.beam.runners.spark.aggregators.NamedAggregators;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
@@ -58,7 +59,6 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.metrics.MetricsContainerStepMap;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.CombineWithContext;
 import org.apache.beam.sdk.transforms.DoFn;
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java
index 86f35ba09559..5728fa044850 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java
@@ -24,8 +24,8 @@
 
 import java.io.IOException;
 import java.util.NoSuchElementException;
+import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.sdk.io.Source;
-import org.apache.beam.sdk.metrics.MetricsContainer;
 import org.joda.time.Instant;
 import org.junit.Rule;
 import org.junit.Test;
@@ -91,7 +91,7 @@ public Source<Integer> getCurrentSource() {
   private final TestReader testReader = new TestReader();
 
   private final SourceRDD.Bounded.ReaderToIteratorAdapter<Integer> readerIterator =
-      new SourceRDD.Bounded.ReaderToIteratorAdapter<>(new MetricsContainer(""), testReader);
+      new SourceRDD.Bounded.ReaderToIteratorAdapter<>(new MetricsContainerImpl(""), testReader);
 
   private void assertReaderRange(final int start, final int end) {
     for (int i = start; i < end; i++) {
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
index 584edace250a..254a318da8d2 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
@@ -17,7 +17,7 @@
  */
 package org.apache.beam.runners.spark.translation.streaming;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.attemptedMetricsResult;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.attemptedMetricsResult;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java
index df6027cf349b..cafd539f7fce 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingSourceMetricsTest.java
@@ -18,7 +18,7 @@
 
 package org.apache.beam.runners.spark.translation.streaming;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.attemptedMetricsResult;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.attemptedMetricsResult;
 import static org.hamcrest.Matchers.hasItem;
 import static org.junit.Assert.assertThat;
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionResult.java
index 27c242c09912..b01ae4646b35 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/DistributionResult.java
@@ -18,10 +18,13 @@
 package org.apache.beam.sdk.metrics;
 
 import com.google.auto.value.AutoValue;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
  * The result of a {@link Distribution} metric.
  */
+@Experimental(Kind.METRICS)
 @AutoValue
 public abstract class DistributionResult {
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeResult.java
index 878776a7887e..f24ded213dc2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/GaugeResult.java
@@ -18,11 +18,14 @@
 package org.apache.beam.sdk.metrics;
 
 import com.google.auto.value.AutoValue;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.joda.time.Instant;
 
 /**
  * The result of a {@link Gauge} metric.
  */
+@Experimental(Kind.METRICS)
 @AutoValue
 public abstract class GaugeResult {
   public abstract long value();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metric.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metric.java
index dcd8a04ec236..fdcc93cd8bc9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metric.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metric.java
@@ -18,10 +18,15 @@
 
 package org.apache.beam.sdk.metrics;
 
+import java.io.Serializable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+
 /**
  * Marker interface for all user-facing metrics.
  */
-public interface Metric {
+@Experimental(Kind.METRICS)
+public interface Metric extends Serializable {
 
   /**
    * The {@link MetricName} given to this metric.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java
index 096d147d1017..bad1f10bc34f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/Metrics.java
@@ -102,7 +102,7 @@ public static Gauge gauge(Class<?> namespace, String name) {
   }
 
   /** Implementation of {@link Counter} that delegates to the instance for the current context. */
-  private static class DelegatingCounter implements Counter, Serializable {
+  private static class DelegatingCounter implements Metric, Counter, Serializable {
     private final MetricName name;
 
     private DelegatingCounter(MetricName name) {
@@ -118,7 +118,7 @@ private DelegatingCounter(MetricName name) {
     @Override public void inc(long n) {
       MetricsContainer container = MetricsEnvironment.getCurrentContainer();
       if (container != null) {
-        container.getCounter(name).update(n);
+        container.getCounter(name).inc(n);
       }
     }
 
@@ -140,7 +140,7 @@ private DelegatingCounter(MetricName name) {
   /**
    * Implementation of {@link Distribution} that delegates to the instance for the current context.
    */
-  private static class DelegatingDistribution implements Distribution, Serializable {
+  private static class DelegatingDistribution implements Metric, Distribution, Serializable {
     private final MetricName name;
 
     private DelegatingDistribution(MetricName name) {
@@ -163,7 +163,7 @@ public void update(long value) {
   /**
    * Implementation of {@link Gauge} that delegates to the instance for the current context.
    */
-  private static class DelegatingGauge implements Gauge, Serializable {
+  private static class DelegatingGauge implements Metric, Gauge, Serializable {
     private final MetricName name;
 
     private DelegatingGauge(MetricName name) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
index 48fa35913971..62b0806906fc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java
@@ -15,164 +15,34 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.sdk.metrics;
 
-import static com.google.common.base.Preconditions.checkNotNull;
+package org.apache.beam.sdk.metrics;
 
-import com.google.common.collect.ImmutableList;
 import java.io.Serializable;
-import java.util.Map;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
 
 /**
  * Holds the metrics for a single step and unit-of-commit (bundle).
- *
- * <p>This class is thread-safe. It is intended to be used with 1 (or more) threads are updating
- * metrics and at-most 1 thread is extracting updates by calling {@link #getUpdates} and
- * {@link #commitUpdates}. Outside of this it is still safe. Although races in the update extraction
- * may cause updates that don't actually have any changes, it will never lose an update.
- *
- * <p>For consistency, all threads that update metrics should finish before getting the final
- * cumulative values/updates.
  */
 @Experimental(Kind.METRICS)
-public class MetricsContainer implements Serializable {
-
-  private final String stepName;
-
-  private MetricsMap<MetricName, CounterCell> counters =
-      new MetricsMap<>(new MetricsMap.Factory<MetricName, CounterCell>() {
-        @Override
-        public CounterCell createInstance(MetricName unusedKey) {
-          return new CounterCell();
-        }
-      });
-
-  private MetricsMap<MetricName, DistributionCell> distributions =
-      new MetricsMap<>(new MetricsMap.Factory<MetricName, DistributionCell>() {
-        @Override
-        public DistributionCell createInstance(MetricName unusedKey) {
-          return new DistributionCell();
-        }
-      });
-
-  private MetricsMap<MetricName, GaugeCell> gauges =
-      new MetricsMap<>(new MetricsMap.Factory<MetricName, GaugeCell>() {
-        @Override
-        public GaugeCell createInstance(MetricName unusedKey) {
-          return new GaugeCell();
-        }
-      });
-
-  /**
-   * Create a new {@link MetricsContainer} associated with the given {@code stepName}.
-   */
-  public MetricsContainer(String stepName) {
-    this.stepName = stepName;
-  }
+public interface MetricsContainer extends Serializable {
 
   /**
-   * Return the {@link CounterCell} that should be used for implementing the given
+   * Return the {@link Counter} that should be used for implementing the given
    * {@code metricName} in this container.
    */
-  public CounterCell getCounter(MetricName metricName) {
-    return counters.get(metricName);
-  }
+  Counter getCounter(MetricName metricName);
 
   /**
-   * Return the {@link DistributionCell} that should be used for implementing the given
+   * Return the {@link Distribution} that should be used for implementing the given
    * {@code metricName} in this container.
    */
-  public DistributionCell getDistribution(MetricName metricName) {
-    return distributions.get(metricName);
-  }
+  Distribution getDistribution(MetricName metricName);
 
   /**
-   * Return the {@link GaugeCell} that should be used for implementing the given
+   * Return the {@link Gauge} that should be used for implementing the given
    * {@code metricName} in this container.
    */
-  public GaugeCell getGauge(MetricName metricName) {
-    return gauges.get(metricName);
-  }
-
-  private <UserT extends Metric, UpdateT, CellT extends MetricCell<UserT, UpdateT>>
-  ImmutableList<MetricUpdate<UpdateT>> extractUpdates(
-      MetricsMap<MetricName, CellT> cells) {
-    ImmutableList.Builder<MetricUpdate<UpdateT>> updates = ImmutableList.builder();
-    for (Map.Entry<MetricName, CellT> cell : cells.entries()) {
-      if (cell.getValue().getDirty().beforeCommit()) {
-        updates.add(MetricUpdate.create(MetricKey.create(stepName, cell.getKey()),
-            cell.getValue().getCumulative()));
-      }
-    }
-    return updates.build();
-  }
-
-  /**
-   * Return the cumulative values for any metrics that have changed since the last time updates were
-   * committed.
-   */
-  public MetricUpdates getUpdates() {
-    return MetricUpdates.create(
-        extractUpdates(counters),
-        extractUpdates(distributions),
-        extractUpdates(gauges));
-  }
-
-  private void commitUpdates(MetricsMap<MetricName, ? extends MetricCell<?, ?>> cells) {
-    for (MetricCell<?, ?> cell : cells.values()) {
-      cell.getDirty().afterCommit();
-    }
-  }
-
-  /**
-   * Mark all of the updates that were retrieved with the latest call to {@link #getUpdates()} as
-   * committed.
-   */
-  public void commitUpdates() {
-    commitUpdates(counters);
-    commitUpdates(distributions);
-    commitUpdates(gauges);
-  }
-
-  private <UserT extends Metric, UpdateT, CellT extends MetricCell<UserT, UpdateT>>
-  ImmutableList<MetricUpdate<UpdateT>> extractCumulatives(
-      MetricsMap<MetricName, CellT> cells) {
-    ImmutableList.Builder<MetricUpdate<UpdateT>> updates = ImmutableList.builder();
-    for (Map.Entry<MetricName, CellT> cell : cells.entries()) {
-      UpdateT update = checkNotNull(cell.getValue().getCumulative());
-      updates.add(MetricUpdate.create(MetricKey.create(stepName, cell.getKey()), update));
-    }
-    return updates.build();
-  }
-
-  /**
-   * Return the {@link MetricUpdates} representing the cumulative values of all metrics in this
-   * container.
-   */
-  public MetricUpdates getCumulative() {
-    return MetricUpdates.create(
-        extractCumulatives(counters),
-        extractCumulatives(distributions),
-        extractCumulatives(gauges));
-  }
-
-  /**
-   * Update values of this {@link MetricsContainer} by merging the value of another cell.
-   */
-  public void update(MetricsContainer other) {
-    updateCells(counters, other.counters);
-    updateCells(distributions, other.distributions);
-    updateCells(gauges, other.gauges);
-  }
-
-  private <UserT extends Metric, DataT, CellT extends MetricCell<UserT, DataT>> void updateCells(
-      MetricsMap<MetricName, CellT> current,
-      MetricsMap<MetricName, CellT> updates) {
-    for (Map.Entry<MetricName, CellT> counter : updates.entries()) {
-      current.get(counter.getKey()).update(counter.getValue());
-    }
-  }
+  Gauge getGauge(MetricName metricName);
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java
index a4b311ffd43e..dadec339283d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsEnvironment.java
@@ -21,6 +21,9 @@
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,7 +33,7 @@
  * <p>Users should not interact directly with this class. Instead, use {@link Metrics} and the
  * returned objects to create and modify metrics.
  *
- * <p>The runner should create {@link MetricsContainer} for each context in which metrics are
+ * <p>The runner should create a {@link MetricsContainer} for each context in which metrics are
  * reported (by step and name) and call {@link #setCurrentContainer} before invoking any code that
  * may update metrics within that step. It should call {@link #setCurrentContainer} again to restore
  * the previous container.
@@ -39,9 +42,11 @@
  * container for the current thread and get a {@link Closeable} that will restore the previous
  * container when closed.
  */
+@Experimental(Kind.METRICS)
+@Internal
 public class MetricsEnvironment {
 
-  private static final Logger LOG = LoggerFactory.getLogger(MetricsContainer.class);
+  private static final Logger LOG = LoggerFactory.getLogger(MetricsEnvironment.class);
 
   private static final AtomicBoolean METRICS_SUPPORTED = new AtomicBoolean(false);
   private static final AtomicBoolean REPORTED_MISSING_CONTAINER = new AtomicBoolean(false);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SinkMetrics.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SinkMetrics.java
index f96b6aced5e5..f6f00bd25038 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SinkMetrics.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SinkMetrics.java
@@ -17,9 +17,13 @@
  */
 package org.apache.beam.sdk.metrics;
 
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+
 /**
  * Standard Sink Metrics.
  */
+@Experimental(Kind.METRICS)
 public class SinkMetrics {
 
   private static final String SINK_NAMESPACE = "sink";
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java
index 4479f3aecd27..1adbf0c4b6bb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java
@@ -18,10 +18,13 @@
 package org.apache.beam.sdk.metrics;
 
 import com.google.common.base.Joiner;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 
 /**
  * Standard {@link org.apache.beam.sdk.io.Source} Metrics.
  */
+@Experimental(Kind.METRICS)
 public class SourceMetrics {
 
   private static final String SOURCE_NAMESPACE = "source";
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricMatchers.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricResultsMatchers.java
similarity index 78%
rename from sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricMatchers.java
rename to sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricResultsMatchers.java
index a0dd1195ec01..50319521595d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricMatchers.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricResultsMatchers.java
@@ -19,66 +19,14 @@
 package org.apache.beam.sdk.metrics;
 
 import java.util.Objects;
-import org.apache.beam.sdk.metrics.MetricUpdates.MetricUpdate;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.TypeSafeMatcher;
 
 /**
- * Matchers for metrics.
+ * Matchers for {@link MetricResults}.
  */
-public class MetricMatchers {
-
-  /**
-   * Matches a {@link MetricUpdate} with the given name and contents.
-   *
-   * <p>Visible since it may be used in runner-specific tests.
-   */
-  public static <T> Matcher<MetricUpdate<T>> metricUpdate(final String name, final T update) {
-    return new TypeSafeMatcher<MetricUpdate<T>>() {
-      @Override
-      protected boolean matchesSafely(MetricUpdate<T> item) {
-        return Objects.equals(name, item.getKey().metricName().name())
-            && Objects.equals(update, item.getUpdate());
-      }
-
-      @Override
-      public void describeTo(Description description) {
-        description
-            .appendText("MetricUpdate{name=").appendValue(name)
-            .appendText(", update=").appendValue(update)
-            .appendText("}");
-      }
-    };
-  }
-
-  /**
-   * Matches a {@link MetricUpdate} with the given namespace, name, step and contents.
-   *
-   * <p>Visible since it may be used in runner-specific tests.
-   */
-  public static <T> Matcher<MetricUpdate<T>> metricUpdate(
-      final String namespace, final String name, final String step, final T update) {
-    return new TypeSafeMatcher<MetricUpdate<T>>() {
-      @Override
-      protected boolean matchesSafely(MetricUpdate<T> item) {
-        return Objects.equals(namespace, item.getKey().metricName().namespace())
-            && Objects.equals(name, item.getKey().metricName().name())
-            && Objects.equals(step, item.getKey().stepName())
-            && Objects.equals(update, item.getUpdate());
-      }
-
-      @Override
-      public void describeTo(Description description) {
-        description
-            .appendText("MetricUpdate{inNamespace=").appendValue(namespace)
-            .appendText(", name=").appendValue(name)
-            .appendText(", step=").appendValue(step)
-            .appendText(", update=").appendValue(update)
-            .appendText("}");
-      }
-    };
-  }
+public class MetricResultsMatchers {
 
   /**
    * Matches a {@link MetricResult} with the given namespace, name and step, and whose value equals
@@ -164,7 +112,7 @@ static Matcher<MetricResult<DistributionResult>> distributionCommittedMinMax(
     return distributionMinMax(namespace, name, step, committedMin, committedMax, true);
   }
 
-  static Matcher<MetricResult<DistributionResult>> distributionMinMax(
+  public static Matcher<MetricResult<DistributionResult>> distributionMinMax(
       final String namespace, final String name, final String step,
       final Long min, final Long max, final boolean isCommitted) {
     final String metricState = isCommitted ? "committed" : "attempted";
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsEnvironmentTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsEnvironmentTest.java
index 0ce17b4bb9c3..a29c13bdc4b2 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsEnvironmentTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsEnvironmentTest.java
@@ -18,15 +18,16 @@
 
 package org.apache.beam.sdk.metrics;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.metricUpdate;
-import static org.hamcrest.Matchers.contains;
 import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
 
 import org.junit.After;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
 
 /**
  * Tests for {@link MetricsEnvironment}.
@@ -41,8 +42,13 @@ public void teardown() {
   @Test
   public void testUsesAppropriateMetricsContainer() {
     Counter counter = Metrics.counter("ns", "name");
-    MetricsContainer c1 = new MetricsContainer("step1");
-    MetricsContainer c2 = new MetricsContainer("step2");
+
+    MetricsContainer c1 = Mockito.mock(MetricsContainer.class);
+    MetricsContainer c2 = Mockito.mock(MetricsContainer.class);
+    Counter counter1 = Mockito.mock(Counter.class);
+    Counter counter2 = Mockito.mock(Counter.class);
+    when(c1.getCounter(MetricName.named("ns", "name"))).thenReturn(counter1);
+    when(c2.getCounter(MetricName.named("ns", "name"))).thenReturn(counter2);
 
     MetricsEnvironment.setCurrentContainer(c1);
     counter.inc();
@@ -50,10 +56,9 @@ public void testUsesAppropriateMetricsContainer() {
     counter.dec();
     MetricsEnvironment.setCurrentContainer(null);
 
-    MetricUpdates updates1 = c1.getUpdates();
-    MetricUpdates updates2 = c2.getUpdates();
-    assertThat(updates1.counterUpdates(), contains(metricUpdate("ns", "name", "step1", 1L)));
-    assertThat(updates2.counterUpdates(), contains(metricUpdate("ns", "name", "step2", -1L)));
+    verify(counter1).inc(1L);
+    verify(counter2).inc(-1L);
+    verifyNoMoreInteractions(counter1, counter2);
   }
 
   @Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
index 084c44508a75..bc768f835e39 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
@@ -18,13 +18,14 @@
 
 package org.apache.beam.sdk.metrics;
 
-import static org.apache.beam.sdk.metrics.MetricMatchers.attemptedMetricsResult;
-import static org.apache.beam.sdk.metrics.MetricMatchers.distributionMinMax;
-import static org.apache.beam.sdk.metrics.MetricMatchers.metricsResult;
-import static org.hamcrest.Matchers.equalTo;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.attemptedMetricsResult;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.distributionMinMax;
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.metricsResult;
 import static org.hamcrest.Matchers.hasItem;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 import java.io.Serializable;
 import org.apache.beam.sdk.PipelineResult;
@@ -41,13 +42,13 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
-import org.hamcrest.CoreMatchers;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 /**
  * Tests for {@link Metrics}.
@@ -95,40 +96,40 @@ public void testCounterWithoutContainer() {
 
   @Test
   public void testDistributionToCell() {
-    MetricsContainer container = new MetricsContainer("step");
-    MetricsEnvironment.setCurrentContainer(container);
+    MetricsContainer mockContainer = Mockito.mock(MetricsContainer.class);
+    Distribution mockDistribution = Mockito.mock(Distribution.class);
+    when(mockContainer.getDistribution(METRIC_NAME)).thenReturn(mockDistribution);
 
     Distribution distribution = Metrics.distribution(NS, NAME);
 
+    MetricsEnvironment.setCurrentContainer(mockContainer);
     distribution.update(5L);
 
-    DistributionCell cell = container.getDistribution(METRIC_NAME);
-    assertThat(cell.getCumulative(), equalTo(DistributionData.create(5, 1, 5, 5)));
+    verify(mockDistribution).update(5L);
 
     distribution.update(36L);
-    assertThat(cell.getCumulative(), equalTo(DistributionData.create(41, 2, 5, 36)));
-
     distribution.update(1L);
-    assertThat(cell.getCumulative(), equalTo(DistributionData.create(42, 3, 1, 36)));
+    verify(mockDistribution).update(36L);
+    verify(mockDistribution).update(1L);
   }
 
   @Test
   public void testCounterToCell() {
-    MetricsContainer container = new MetricsContainer("step");
-    MetricsEnvironment.setCurrentContainer(container);
+    MetricsContainer mockContainer = Mockito.mock(MetricsContainer.class);
+    Counter mockCounter = Mockito.mock(Counter.class);
+    when(mockContainer.getCounter(METRIC_NAME)).thenReturn(mockCounter);
+
     Counter counter = Metrics.counter(NS, NAME);
-    CounterCell cell = container.getCounter(METRIC_NAME);
+
+    MetricsEnvironment.setCurrentContainer(mockContainer);
     counter.inc();
-    assertThat(cell.getCumulative(), CoreMatchers.equalTo(1L));
+    verify(mockCounter).inc(1);
 
     counter.inc(47L);
-    assertThat(cell.getCumulative(), CoreMatchers.equalTo(48L));
+    verify(mockCounter).inc(47);
 
     counter.dec(5L);
-    assertThat(cell.getCumulative(), CoreMatchers.equalTo(43L));
-
-    counter.dec();
-    assertThat(cell.getCumulative(), CoreMatchers.equalTo(42L));
+    verify(mockCounter).inc(-5);
   }
 
   @Category({ValidatesRunner.class, UsesCommittedMetrics.class, UsesCounterMetrics.class,
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
index 12b7c78fd8cd..691f7f4fdc4f 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.io.kafka;
 
+import static org.apache.beam.sdk.metrics.MetricResultsMatchers.attemptedMetricsResult;
 import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
 import static org.hamcrest.Matchers.hasItem;
 import static org.junit.Assert.assertEquals;
@@ -54,7 +55,6 @@
 import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader;
 import org.apache.beam.sdk.io.kafka.serialization.InstantDeserializer;
 import org.apache.beam.sdk.metrics.GaugeResult;
-import org.apache.beam.sdk.metrics.MetricMatchers;
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.metrics.MetricNameFilter;
 import org.apache.beam.sdk.metrics.MetricQueryResults;
@@ -587,33 +587,29 @@ public void testUnboundedSourceMetrics() {
 
     Iterable<MetricResult<Long>> counters = metrics.counters();
 
-    assertThat(counters, hasItem(
-        MetricMatchers.attemptedMetricsResult(
-            elementsRead.namespace(),
-            elementsRead.name(),
-            readStep,
-            1000L)));
-
-    assertThat(counters, hasItem(
-        MetricMatchers.attemptedMetricsResult(
-            elementsReadBySplit.namespace(),
-            elementsReadBySplit.name(),
-            readStep,
-            1000L)));
-
-    assertThat(counters, hasItem(
-        MetricMatchers.attemptedMetricsResult(
-            bytesRead.namespace(),
-            bytesRead.name(),
-            readStep,
-            12000L)));
-
-    assertThat(counters, hasItem(
-        MetricMatchers.attemptedMetricsResult(
-            bytesReadBySplit.namespace(),
-            bytesReadBySplit.name(),
-            readStep,
-            12000L)));
+    assertThat(counters, hasItem(attemptedMetricsResult(
+        elementsRead.namespace(),
+        elementsRead.name(),
+        readStep,
+        1000L)));
+
+    assertThat(counters, hasItem(attemptedMetricsResult(
+        elementsReadBySplit.namespace(),
+        elementsReadBySplit.name(),
+        readStep,
+        1000L)));
+
+    assertThat(counters, hasItem(attemptedMetricsResult(
+        bytesRead.namespace(),
+        bytesRead.name(),
+        readStep,
+        12000L)));
+
+    assertThat(counters, hasItem(attemptedMetricsResult(
+        bytesReadBySplit.namespace(),
+        bytesReadBySplit.name(),
+        readStep,
+        12000L)));
 
     MetricQueryResults backlogElementsMetrics =
         result.metrics().queryMetrics(
@@ -912,7 +908,7 @@ public void testSinkMetrics() throws Exception {
 
 
       assertThat(metrics.counters(), hasItem(
-          MetricMatchers.attemptedMetricsResult(
+          attemptedMetricsResult(
               elementsWritten.namespace(),
               elementsWritten.name(),
               "writeToKafka",

From 416b89b7ff1bd26e7d9a1b3c29f294d2ae922238 Mon Sep 17 00:00:00 2001
From: Jason Kuster <jasonkuster@google.com>
Date: Wed, 10 May 2017 17:55:53 -0700
Subject: [PATCH 190/387] bump time of precommits

Signed-off-by: Jason Kuster <jasonkuster@google.com>
---
 .../jenkins/job_beam_PreCommit_Java_MavenInstall.groovy      | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
index 371855159984..bc130ece20c0 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
@@ -29,7 +29,10 @@ mavenJob('beam_PreCommit_Java_MavenInstall') {
   concurrentBuild()
 
   // Set common parameters.
-  common_job_properties.setTopLevelMainJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(
+    delegate,
+    'master',
+    120)
 
   // Set Maven parameters.
   common_job_properties.setMavenConfig(delegate)

From 316ff6bf9f472687667482805062b0aa6417aa8f Mon Sep 17 00:00:00 2001
From: Devon Meunier <devon.meunier@shopify.com>
Date: Wed, 3 May 2017 16:22:16 -0400
Subject: [PATCH 191/387] [BEAM-2150] Relax regex to support wildcard globbing
 for GCS

---
 .../extensions/gcp/storage/GcsFileSystem.java |  6 +-
 .../gcp/storage/GcsPathValidator.java         |  4 +-
 .../org/apache/beam/sdk/util/GcsUtil.java     | 46 ++++-----------
 .../gcp/storage/GcsFileSystemTest.java        |  8 ---
 .../org/apache/beam/sdk/util/GcsUtilTest.java | 57 +++++++++++++++----
 5 files changed, 63 insertions(+), 58 deletions(-)

diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
index 9052a5ac12cc..6db0a01e847d 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java
@@ -73,7 +73,7 @@ protected List<MatchResult> match(List<String> specs) throws IOException {
     List<Boolean> isGlobBooleans = Lists.newArrayList();
 
     for (GcsPath path : gcsPaths) {
-      if (GcsUtil.isGlob(path)) {
+      if (GcsUtil.isWildcard(path)) {
         globs.add(path);
         isGlobBooleans.add(true);
       } else {
@@ -178,8 +178,8 @@ public MatchResult apply(GcsPath gcsPath) {
    */
   @VisibleForTesting
   MatchResult expand(GcsPath gcsPattern) throws IOException {
-    String prefix = GcsUtil.getGlobPrefix(gcsPattern.getObject());
-    Pattern p = Pattern.compile(GcsUtil.globToRegexp(gcsPattern.getObject()));
+    String prefix = GcsUtil.getNonWildcardPrefix(gcsPattern.getObject());
+    Pattern p = Pattern.compile(GcsUtil.wildcardToRegexp(gcsPattern.getObject()));
 
     LOG.debug("matching files in bucket {}, prefix {} against pattern {}", gcsPattern.getBucket(),
         prefix, p.toString());
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java
index e7257b24b8cc..6b10bfba1e0b 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java
@@ -23,7 +23,6 @@
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 
 /**
@@ -47,8 +46,7 @@ public static GcsPathValidator fromOptions(PipelineOptions options) {
    */
   @Override
   public void validateInputFilePatternSupported(String filepattern) {
-    GcsPath gcsPath = getGcsPath(filepattern);
-    checkArgument(GcsUtil.isGcsPatternSupported(gcsPath.getObject()));
+    getGcsPath(filepattern);
     verifyPath(filepattern);
     verifyPathIsAccessible(filepattern, "Could not find file %s");
   }
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
index 18e3e2bce080..94b733a7f8dc 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
@@ -125,14 +125,6 @@ public static GcsUtil create(
   /** Matches a glob containing a wildcard, capturing the portion before the first wildcard. */
   private static final Pattern GLOB_PREFIX = Pattern.compile("(?<PREFIX>[^\\[*?]*)[\\[*?].*");
 
-  private static final String RECURSIVE_WILDCARD = "[*]{2}";
-
-  /**
-   * A {@link Pattern} for globs with a recursive wildcard.
-   */
-  private static final Pattern RECURSIVE_GCS_PATTERN =
-      Pattern.compile(".*" + RECURSIVE_WILDCARD + ".*");
-
   /**
    * Maximum number of requests permitted in a GCS batch request.
    */
@@ -159,23 +151,10 @@ public static GcsUtil create(
   // Exposed for testing.
   final ExecutorService executorService;
 
-  /**
-   * Returns true if the given GCS pattern is supported otherwise fails with an
-   * exception.
-   */
-  public static boolean isGcsPatternSupported(String gcsPattern) {
-    if (RECURSIVE_GCS_PATTERN.matcher(gcsPattern).matches()) {
-      throw new IllegalArgumentException("Unsupported wildcard usage in \"" + gcsPattern + "\": "
-          + " recursive wildcards are not supported.");
-    }
-    return true;
-  }
-
   /**
    * Returns the prefix portion of the glob that doesn't contain wildcards.
    */
-  public static String getGlobPrefix(String globExp) {
-    checkArgument(isGcsPatternSupported(globExp));
+  public static String getNonWildcardPrefix(String globExp) {
     Matcher m = GLOB_PREFIX.matcher(globExp);
     checkArgument(
         m.matches(),
@@ -189,15 +168,15 @@ public static String getGlobPrefix(String globExp) {
    * @param globExp the glob expression to expand
    * @return a string with the regular expression this glob expands to
    */
-  public static String globToRegexp(String globExp) {
+  public static String wildcardToRegexp(String globExp) {
     StringBuilder dst = new StringBuilder();
-    char[] src = globExp.toCharArray();
+    char[] src = globExp.replace("**/*", "**").toCharArray();
     int i = 0;
     while (i < src.length) {
       char c = src[i++];
       switch (c) {
         case '*':
-          dst.append("[^/]*");
+          dst.append(".*");
           break;
         case '?':
           dst.append("[^/]");
@@ -226,9 +205,9 @@ public static String globToRegexp(String globExp) {
   }
 
   /**
-   * Returns true if the given {@code spec} contains glob.
+   * Returns true if the given {@code spec} contains wildcard.
    */
-  public static boolean isGlob(GcsPath spec) {
+  public static boolean isWildcard(GcsPath spec) {
     return GLOB_PREFIX.matcher(spec.getObject()).matches();
   }
 
@@ -254,11 +233,14 @@ protected void setStorageClient(Storage storageClient) {
    * exists.
    */
   public List<GcsPath> expand(GcsPath gcsPattern) throws IOException {
-    checkArgument(isGcsPatternSupported(gcsPattern.getObject()));
     Pattern p = null;
     String prefix = null;
-    if (!isGlob(gcsPattern)) {
-      // Not a glob.
+    if (isWildcard(gcsPattern)) {
+      // Part before the first wildcard character.
+      prefix = getNonWildcardPrefix(gcsPattern.getObject());
+      p = Pattern.compile(wildcardToRegexp(gcsPattern.getObject()));
+    } else {
+      // Not a wildcard.
       try {
         // Use a get request to fetch the metadata of the object, and ignore the return value.
         // The request has strong global consistency.
@@ -268,10 +250,6 @@ public List<GcsPath> expand(GcsPath gcsPattern) throws IOException {
         // If the path was not found, return an empty list.
         return ImmutableList.of();
       }
-    } else {
-      // Part before the first wildcard character.
-      prefix = getGlobPrefix(gcsPattern.getObject());
-      p = Pattern.compile(globToRegexp(gcsPattern.getObject()));
     }
 
     LOG.debug("matching files in bucket {}, prefix {} against pattern {}", gcsPattern.getBucket(),
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java
index 03194d28a77b..ce6bbba648bc 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java
@@ -214,14 +214,6 @@ public void testExpandNonGlob() throws Exception {
     gcsFileSystem.expand(GcsPath.fromUri("gs://testbucket/testdirectory/otherfile"));
   }
 
-  // Patterns that contain recursive wildcards ('**') are not supported.
-  @Test
-  public void testRecursiveGlobExpansionFails() throws IOException {
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("Unsupported wildcard usage");
-    gcsFileSystem.expand(GcsPath.fromUri("gs://testbucket/test**"));
-  }
-
   @Test
   public void testMatchNonGlobs() throws Exception {
     List<StorageObjectOrIOException> items = new ArrayList<>();
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
index 0af584e74e67..532645047d5a 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
@@ -92,10 +92,12 @@ public class GcsUtilTest {
 
   @Test
   public void testGlobTranslation() {
-    assertEquals("foo", GcsUtil.globToRegexp("foo"));
-    assertEquals("fo[^/]*o", GcsUtil.globToRegexp("fo*o"));
-    assertEquals("f[^/]*o\\.[^/]", GcsUtil.globToRegexp("f*o.?"));
-    assertEquals("foo-[0-9][^/]*", GcsUtil.globToRegexp("foo-[0-9]*"));
+    assertEquals("foo", GcsUtil.wildcardToRegexp("foo"));
+    assertEquals("fo.*o", GcsUtil.wildcardToRegexp("fo*o"));
+    assertEquals("f.*o\\.[^/]", GcsUtil.wildcardToRegexp("f*o.?"));
+    assertEquals("foo-[0-9].*", GcsUtil.wildcardToRegexp("foo-[0-9]*"));
+    assertEquals(".*.*foo", GcsUtil.wildcardToRegexp("**/*foo"));
+    assertEquals(".*.*foo", GcsUtil.wildcardToRegexp("**foo"));
   }
 
   private static GcsOptions gcsOptionsWithTestCredential() {
@@ -260,16 +262,51 @@ public void testGlobExpansion() throws IOException {
     }
   }
 
-  // Patterns that contain recursive wildcards ('**') are not supported.
   @Test
-  public void testRecursiveGlobExpansionFails() throws IOException {
+  public void testRecursiveGlobExpansion() throws IOException {
     GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
     GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
-    GcsPath pattern = GcsPath.fromUri("gs://testbucket/test**");
 
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("Unsupported wildcard usage");
-    gcsUtil.expand(pattern);
+    Storage mockStorage = Mockito.mock(Storage.class);
+    gcsUtil.setStorageClient(mockStorage);
+
+    Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
+    Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);
+    Storage.Objects.List mockStorageList = Mockito.mock(Storage.Objects.List.class);
+
+    Objects modelObjects = new Objects();
+    List<StorageObject> items = new ArrayList<>();
+    // A directory
+    items.add(new StorageObject().setBucket("testbucket").setName("testdirectory/"));
+
+    // Files within the directory
+    items.add(new StorageObject().setBucket("testbucket").setName("test/directory/file1.txt"));
+    items.add(new StorageObject().setBucket("testbucket").setName("test/directory/file2.txt"));
+    items.add(new StorageObject().setBucket("testbucket").setName("test/directory/file3.txt"));
+    items.add(new StorageObject().setBucket("testbucket").setName("test/directory/otherfile"));
+    items.add(new StorageObject().setBucket("testbucket").setName("test/directory/anotherfile"));
+    items.add(new StorageObject().setBucket("testbucket").setName("test/file4.txt"));
+
+    modelObjects.setItems(items);
+
+    when(mockStorage.objects()).thenReturn(mockStorageObjects);
+    when(mockStorageObjects.get("testbucket", "test/directory/otherfile")).thenReturn(
+        mockStorageGet);
+    when(mockStorageObjects.list("testbucket")).thenReturn(mockStorageList);
+    when(mockStorageGet.execute()).thenReturn(
+        new StorageObject().setBucket("testbucket").setName("test/directory/otherfile"));
+    when(mockStorageList.execute()).thenReturn(modelObjects);
+
+    {
+      GcsPath pattern = GcsPath.fromUri("gs://testbucket/test/**/*.txt");
+      List<GcsPath> expectedFiles = ImmutableList.of(
+          GcsPath.fromUri("gs://testbucket/test/directory/file1.txt"),
+          GcsPath.fromUri("gs://testbucket/test/directory/file2.txt"),
+          GcsPath.fromUri("gs://testbucket/test/directory/file3.txt"),
+          GcsPath.fromUri("gs://testbucket/test/file4.txt"));
+
+      assertThat(expectedFiles, contains(gcsUtil.expand(pattern).toArray()));
+    }
   }
 
   // GCSUtil.expand() should fail when matching a single object when that object does not exist.

From 86486e279e952011865769c6e53cf348e52914c0 Mon Sep 17 00:00:00 2001
From: Borisa Zivkovic <borisa.zivkovic@huawei.com>
Date: Thu, 11 May 2017 10:27:41 +0100
Subject: [PATCH 192/387] Add support for local execution to PubsubIO using the
 google cloud emulator

---
 .../java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index 048fded462b2..bdc2752e7fa9 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -65,6 +65,12 @@
  * Read and Write {@link PTransform}s for Cloud Pub/Sub streams. These transforms create
  * and consume unbounded {@link PCollection PCollections}.
  *
+ * <h3>Using local emulator</h3>
+ *
+ * <p>In order to use local emulator for Pubsub you should use
+ * {@code PubsubOptions#setPubsubRootUrl(String)} method to set host and port of your
+ * local emulator.
+ *
  * <h3>Permissions</h3>
  *
  * <p>Permission requirements depend on the {@link PipelineRunner} that is used to execute the

From 875735aec9b402be50969ce37d396891730b29f0 Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Thu, 11 May 2017 16:07:10 +0800
Subject: [PATCH 193/387] minor typo fix in comment

---
 .../src/main/java/org/apache/beam/sdk/transforms/Top.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
index dd8bc5f6f659..99ec49bb8ef8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
@@ -66,7 +66,7 @@ private Top() {
    * given {@code Comparator<T>}.  The {@code Comparator<T>} must also
    * be {@code Serializable}.
    *
-   * <p>If {@code count} {@code <} the number of elements in the
+   * <p>If {@code count} {@code >} the number of elements in the
    * input {@code PCollection}, then all the elements of the input
    * {@code PCollection} will be in the resulting
    * {@code List}, albeit in sorted order.
@@ -110,7 +110,7 @@ Combine.Globally<T, List<T>> of(int count, ComparatorT compareFn) {
    * {@code PCollection<T>}, in increasing order, sorted according to
    * their natural order.
    *
-   * <p>If {@code count} {@code <} the number of elements in the
+   * <p>If {@code count} {@code >} the number of elements in the
    * input {@code PCollection}, then all the elements of the input
    * {@code PCollection} will be in the resulting {@code PCollection}'s
    * {@code List}, albeit in sorted order.
@@ -154,7 +154,7 @@ public static <T extends Comparable<T>> Combine.Globally<T, List<T>> smallest(in
    * {@code PCollection<T>}, in decreasing order, sorted according to
    * their natural order.
    *
-   * <p>If {@code count} {@code <} the number of elements in the
+   * <p>If {@code count} {@code >} the number of elements in the
    * input {@code PCollection}, then all the elements of the input
    * {@code PCollection} will be in the resulting {@code PCollection}'s
    * {@code List}, albeit in sorted order.

From aee9187746a1dc0f10865e803a9eb57bdf296686 Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Wed, 10 May 2017 17:39:01 -0700
Subject: [PATCH 194/387] Remove protobuf and http-client dependency from
 runners/google-cloud-dataflow

---
 runners/google-cloud-dataflow-java/pom.xml           | 11 -----------
 .../runners/dataflow/internal/CustomSources.java     | 12 ------------
 2 files changed, 23 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index f67997652853..073cee8085a5 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -293,12 +293,6 @@
       <artifactId>google-http-client-jackson2</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client-protobuf</artifactId>
-      <scope>runtime</scope>
-    </dependency>
-
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-dataflow</artifactId>
@@ -352,11 +346,6 @@
       <artifactId>joda-time</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.protobuf</groupId>
-      <artifactId>protobuf-java</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
index 0d9356655928..ad2a6668daa2 100755
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
@@ -25,7 +25,6 @@
 
 import com.google.api.services.dataflow.model.SourceMetadata;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ByteString;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
@@ -47,20 +46,9 @@
 public class CustomSources {
   private static final String SERIALIZED_SOURCE = "serialized_source";
   @VisibleForTesting static final String SERIALIZED_SOURCE_SPLITS = "serialized_source_splits";
-  /**
-   * The current limit on the size of a ReportWorkItemStatus RPC to Google Cloud Dataflow, which
-   * includes the initial splits, is 20 MB.
-   */
-  public static final long DATAFLOW_SPLIT_RESPONSE_API_SIZE_BYTES = 20 * (1 << 20);
 
   private static final Logger LOG = LoggerFactory.getLogger(CustomSources.class);
 
-  private static final ByteString firstSplitKey = ByteString.copyFromUtf8("0000000000000001");
-
-  public static boolean isFirstUnboundedSourceSplit(ByteString splitKey) {
-    return splitKey.equals(firstSplitKey);
-  }
-
   private static int getDesiredNumUnboundedSourceSplits(DataflowPipelineOptions options) {
     int cores = 4; //TODO: decide at runtime?
     if (options.getMaxNumWorkers() > 0) {

From 0fa97ab789d7ec29f5370d9a25210f1160e75054 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Thu, 11 May 2017 11:34:59 -0700
Subject: [PATCH 195/387] Add internal usage only comments to util/

---
 sdks/python/apache_beam/utils/__init__.py    | 5 ++++-
 sdks/python/apache_beam/utils/annotations.py | 4 +++-
 sdks/python/apache_beam/utils/counters.py    | 5 ++++-
 sdks/python/apache_beam/utils/processes.py   | 6 +++++-
 sdks/python/apache_beam/utils/profiler.py    | 5 ++++-
 sdks/python/apache_beam/utils/proto_utils.py | 2 ++
 sdks/python/apache_beam/utils/retry.py       | 2 ++
 sdks/python/apache_beam/utils/timestamp.py   | 5 ++++-
 sdks/python/apache_beam/utils/urns.py        | 2 ++
 9 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/sdks/python/apache_beam/utils/__init__.py b/sdks/python/apache_beam/utils/__init__.py
index 74cf45d538a7..635c80f7c6b4 100644
--- a/sdks/python/apache_beam/utils/__init__.py
+++ b/sdks/python/apache_beam/utils/__init__.py
@@ -15,4 +15,7 @@
 # limitations under the License.
 #
 
-"""A package containing utilities."""
+"""A package containing internal utilities.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
diff --git a/sdks/python/apache_beam/utils/annotations.py b/sdks/python/apache_beam/utils/annotations.py
index 92318b1420c4..017dd6b81a4d 100644
--- a/sdks/python/apache_beam/utils/annotations.py
+++ b/sdks/python/apache_beam/utils/annotations.py
@@ -15,7 +15,9 @@
 # limitations under the License.
 #
 
-""" Deprecated and experimental annotations.
+"""Deprecated and experimental annotations.
+
+For internal use only; no backwards-compatibility guarantees.
 
 Annotations come in two flavors: deprecated and experimental
 
diff --git a/sdks/python/apache_beam/utils/counters.py b/sdks/python/apache_beam/utils/counters.py
index e41d732a02f6..b3794618ff85 100644
--- a/sdks/python/apache_beam/utils/counters.py
+++ b/sdks/python/apache_beam/utils/counters.py
@@ -18,7 +18,10 @@
 # cython: profile=False
 # cython: overflowcheck=True
 
-"""Counters collect the progress of the Worker for reporting to the service."""
+"""Counters collect the progress of the Worker for reporting to the service.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 import threading
 from apache_beam.transforms import cy_combiners
diff --git a/sdks/python/apache_beam/utils/processes.py b/sdks/python/apache_beam/utils/processes.py
index e089090d5af2..e5fd9c84aaba 100644
--- a/sdks/python/apache_beam/utils/processes.py
+++ b/sdks/python/apache_beam/utils/processes.py
@@ -14,7 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""Cross-platform utilities for creating subprocesses."""
+
+"""Cross-platform utilities for creating subprocesses.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 import platform
 import subprocess
diff --git a/sdks/python/apache_beam/utils/profiler.py b/sdks/python/apache_beam/utils/profiler.py
index 852b65933d17..a2c3f6ab1af6 100644
--- a/sdks/python/apache_beam/utils/profiler.py
+++ b/sdks/python/apache_beam/utils/profiler.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""A profiler context manager based on cProfile.Profile objects."""
+"""A profiler context manager based on cProfile.Profile objects.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 import cProfile
 import logging
diff --git a/sdks/python/apache_beam/utils/proto_utils.py b/sdks/python/apache_beam/utils/proto_utils.py
index d929a92533e6..090a8212f4fc 100644
--- a/sdks/python/apache_beam/utils/proto_utils.py
+++ b/sdks/python/apache_beam/utils/proto_utils.py
@@ -15,6 +15,8 @@
 # limitations under the License.
 #
 
+"""For internal use only; no backwards-compatibility guarantees."""
+
 from google.protobuf import any_pb2
 from google.protobuf import struct_pb2
 
diff --git a/sdks/python/apache_beam/utils/retry.py b/sdks/python/apache_beam/utils/retry.py
index 2c32f0f775d1..1a8b90714d74 100644
--- a/sdks/python/apache_beam/utils/retry.py
+++ b/sdks/python/apache_beam/utils/retry.py
@@ -17,6 +17,8 @@
 
 """Retry decorators for calls raising exceptions.
 
+For internal use only; no backwards-compatibility guarantees.
+
 This module is used mostly to decorate all integration points where the code
 makes calls to remote services. Searching through the code base for @retry
 should find all such places. For this reason even places where retry is not
diff --git a/sdks/python/apache_beam/utils/timestamp.py b/sdks/python/apache_beam/utils/timestamp.py
index 8b2ccda1f168..5d1b48c14e33 100644
--- a/sdks/python/apache_beam/utils/timestamp.py
+++ b/sdks/python/apache_beam/utils/timestamp.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""Timestamp utilities."""
+"""Timestamp utilities.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 from __future__ import absolute_import
 
diff --git a/sdks/python/apache_beam/utils/urns.py b/sdks/python/apache_beam/utils/urns.py
index 46bd8f58f3c9..379b5ff9f20b 100644
--- a/sdks/python/apache_beam/utils/urns.py
+++ b/sdks/python/apache_beam/utils/urns.py
@@ -15,6 +15,8 @@
 # limitations under the License.
 #
 
+"""For internal use only; no backwards-compatibility guarantees."""
+
 import abc
 import inspect
 

From c4d7ece24fafbd3bdcc26d167058e3db2717421a Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 11 May 2017 11:58:08 -0700
Subject: [PATCH 196/387] [BEAM-1345] Mark windowed value as experimental

---
 sdks/python/apache_beam/utils/windowed_value.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/utils/windowed_value.py b/sdks/python/apache_beam/utils/windowed_value.py
index 87c26d128467..be2785432a14 100644
--- a/sdks/python/apache_beam/utils/windowed_value.py
+++ b/sdks/python/apache_beam/utils/windowed_value.py
@@ -16,10 +16,12 @@
 #
 
 """Core windowing data structures.
+
+This module is experimental. No backwards-compatibility guarantees.
 """
 
 # This module is carefully crafted to have optimal performance when
-# compiled whiel still being valid Python.  Care needs to be taken when
+# compiled while still being valid Python.  Care needs to be taken when
 # editing this file as WindowedValues are created for every element for
 # every step in a Beam pipeline.
 

From 67831c39c38a5f040d3c5f21dfbb4be34c88be09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jean-Baptiste=20Onofr=C3=A9?= <jbonofre@apache.org>
Date: Fri, 5 May 2017 18:00:30 +0200
Subject: [PATCH 197/387] [BEAM-2179] Archetype generate-sources.sh cleanup the
 existing sources before rsync

---
 .../examples-java8/generate-sources.sh               | 12 +++++++-----
 .../maven-archetypes/examples/generate-sources.sh    |  8 +++++---
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/sdks/java/maven-archetypes/examples-java8/generate-sources.sh b/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
index 7b85e854df37..58c5f22298b7 100755
--- a/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
+++ b/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
@@ -31,19 +31,21 @@ JAVA8_EXAMPLES_ROOT="${HERE}/../../../../examples/java8"
 # The root of the examples archetype
 ARCHETYPE_ROOT="${HERE}/src/main/resources/archetype-resources"
 
-mkdir -p "$ARCHETYPE_ROOT/src/main/java"
-mkdir -p "$ARCHETYPE_ROOT/src/test/java"
+mkdir -p "${ARCHETYPE_ROOT}/src/main/java"
+mkdir -p "${ARCHETYPE_ROOT}/src/test/java"
 
 #
 # Copy the Java 7 subset of the examples project verbatim. 
 #
 rsync -a --exclude cookbook --exclude complete                  \
     "${EXAMPLES_ROOT}"/src/main/java/org/apache/beam/examples/  \
-    "${ARCHETYPE_ROOT}/src/main/java"
+    "${ARCHETYPE_ROOT}/src/main/java"				            \
+    --delete
 
 rsync -a --exclude cookbook --exclude complete --exclude '*IT.java'  \
-    "${EXAMPLES_ROOT}"/src/test/java/org/apache/beam/examples/        \
-    "${ARCHETYPE_ROOT}/src/test/java"
+    "${EXAMPLES_ROOT}"/src/test/java/org/apache/beam/examples/       \
+    "${ARCHETYPE_ROOT}/src/test/java"				                 \
+    --delete
 
 #
 # Copy in MinimalWordCountJava8 and mobile gaming example
diff --git a/sdks/java/maven-archetypes/examples/generate-sources.sh b/sdks/java/maven-archetypes/examples/generate-sources.sh
index b1a86a7c06be..d9109ac313c2 100755
--- a/sdks/java/maven-archetypes/examples/generate-sources.sh
+++ b/sdks/java/maven-archetypes/examples/generate-sources.sh
@@ -37,11 +37,13 @@ mkdir -p "${ARCHETYPE_ROOT}/src/test/java"
 #
 rsync -a --exclude cookbook --exclude complete                  \
     "${EXAMPLES_ROOT}"/src/main/java/org/apache/beam/examples/  \
-    "${ARCHETYPE_ROOT}/src/main/java"
+    "${ARCHETYPE_ROOT}/src/main/java"				            \
+    --delete
 
 rsync -a --exclude cookbook --exclude complete --exclude '*IT.java'  \
-    "${EXAMPLES_ROOT}"/src/test/java/org/apache/beam/examples/        \
-    "${ARCHETYPE_ROOT}/src/test/java"
+    "${EXAMPLES_ROOT}"/src/test/java/org/apache/beam/examples/       \
+    "${ARCHETYPE_ROOT}/src/test/java"				                 \
+    --delete
 
 #
 # Replace 'package org.apache.beam.examples' with 'package ${package}' in all Java code

From 711f79b338b260185475df6d54af898f7dc51b9d Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 11 May 2017 10:45:20 -0700
Subject: [PATCH 198/387] [BEAM-2260] Improve construction-time errors for Text
 and AvroIO

---
 .../main/java/org/apache/beam/sdk/io/AvroIO.java |  2 ++
 .../main/java/org/apache/beam/sdk/io/TextIO.java |  2 ++
 .../java/org/apache/beam/sdk/io/AvroIOTest.java  | 16 ++++++++++++++++
 .../java/org/apache/beam/sdk/io/TextIOTest.java  | 12 +++++++++++-
 4 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index c7e72336e0c4..d13c6ffb81c4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -441,6 +441,8 @@ public PDone expand(PCollection<T> input) {
           "Cannot set a filename policy and also a filename template or suffix.");
       checkState(getSchema() != null,
           "Need to set the schema of an AvroIO.Write transform.");
+      checkState(!getWindowedWrites() || (getFilenamePolicy() != null),
+          "When using windowed writes, a filename policy must be set via withFilenamePolicy().");
 
       FilenamePolicy usedFilenamePolicy = getFilenamePolicy();
       if (usedFilenamePolicy == null) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
index dbfaeeed8fe5..af6a06967fc7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
@@ -430,6 +430,8 @@ public PDone expand(PCollection<String> input) {
           (getFilenamePolicy() == null)
               || (getShardTemplate() == null && getFilenameSuffix() == null),
           "Cannot set a filename policy and also a filename template or suffix.");
+      checkState(!getWindowedWrites() || (getFilenamePolicy() != null),
+          "When using windowed writes, a filename policy must be set via withFilenamePolicy().");
 
       FilenamePolicy usedFilenamePolicy = getFilenamePolicy();
       if (usedFilenamePolicy == null) {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
index 47b847fc153e..d71f2f74ccd4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
@@ -54,6 +54,7 @@
 import org.apache.avro.reflect.ReflectDatumReader;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.DefaultCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
@@ -77,6 +78,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -93,6 +95,9 @@ public class AvroIOTest {
   @Rule
   public TemporaryFolder tmpFolder = new TemporaryFolder();
 
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
   @Test
   public void testAvroIOGetName() {
     assertEquals("AvroIO.Read", AvroIO.read(String.class).from("/tmp/foo*/baz").getName());
@@ -560,4 +565,15 @@ public void testWriteDisplayData() {
     assertThat(displayData, hasDisplayItem("numShards", 100));
     assertThat(displayData, hasDisplayItem("codec", CodecFactory.snappyCodec().toString()));
   }
+
+  @Test
+  public void testWindowedWriteRequiresFilenamePolicy() {
+    PCollection<String> emptyInput = p.apply(Create.empty(StringUtf8Coder.of()));
+    AvroIO.Write write = AvroIO.write(String.class).to("/tmp/some/file").withWindowedWrites();
+
+    expectedException.expect(IllegalStateException.class);
+    expectedException.expectMessage(
+        "When using windowed writes, a filename policy must be set via withFilenamePolicy()");
+    emptyInput.apply(write);
+  }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
index 685da829c6c2..0d8fbbd34fa3 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
@@ -107,7 +107,6 @@
 @RunWith(JUnit4.class)
 @SuppressWarnings("unchecked")
 public class TextIOTest {
-
   private static final String MY_HEADER = "myHeader";
   private static final String MY_FOOTER = "myFooter";
   private static final String[] EMPTY = new String[] {};
@@ -1103,5 +1102,16 @@ public void testInitialSplitGzipModeGz() throws Exception {
     assertThat(splits, hasSize(equalTo(1)));
     SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
   }
+
+  @Test
+  public void testWindowedWriteRequiresFilenamePolicy() {
+    PCollection<String> emptyInput = p.apply(Create.empty(StringUtf8Coder.of()));
+    TextIO.Write write = TextIO.write().to("/tmp/some/file").withWindowedWrites();
+
+    expectedException.expect(IllegalStateException.class);
+    expectedException.expectMessage(
+        "When using windowed writes, a filename policy must be set via withFilenamePolicy()");
+    emptyInput.apply(write);
+  }
 }
 

From f703dbce7365cc411c941da0dfe67c5398c2b5b6 Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Thu, 11 May 2017 11:33:24 -0700
Subject: [PATCH 199/387] Mark internal modules in python datastoreio

---
 .../apache_beam/io/gcp/datastore/v1/fake_datastore.py      | 7 +++++--
 sdks/python/apache_beam/io/gcp/datastore/v1/helper.py      | 5 ++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py b/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
index bc4d07f24a24..0caf6d612333 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
@@ -15,7 +15,11 @@
 # limitations under the License.
 #
 
-"""Fake datastore used for unit testing."""
+"""Fake datastore used for unit testing.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
+
 import uuid
 
 # Protect against environments where datastore library is not available.
@@ -27,7 +31,6 @@
   pass
 # pylint: enable=wrong-import-order, wrong-import-position
 
-
 def create_run_query(entities, batch_size):
   """A fake datastore run_query method that returns entities in batches.
 
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
index a61884f8cc97..9e2c0531e540 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""Cloud Datastore helper functions."""
+"""Cloud Datastore helper functions.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 import sys
 
 # Protect against environments where datastore library is not available.

From bbe31fc23b73c9cba9fbcf2526f8726f7b0465e1 Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Thu, 11 May 2017 13:31:18 -0700
Subject: [PATCH 200/387] [BEAM-1345] Clearly delineate public API in
 apache_beam/options

---
 .../apache_beam/options/pipeline_options.py   | 20 ++++++++++++++++---
 .../options/pipeline_options_validator.py     |  2 ++
 .../apache_beam/options/value_provider.py     |  8 ++++++++
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py
index b79d85db8595..983d128fd2bf 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -25,6 +25,20 @@
 from apache_beam.options.value_provider import ValueProvider
 
 
+__all__ = [
+    'PipelineOptions',
+    'StandardOptions',
+    'TypeOptions',
+    'DirectOptions',
+    'GoogleCloudOptions',
+    'WorkerOptions',
+    'DebugOptions',
+    'ProfilingOptions',
+    'SetupOptions',
+    'TestOptions',
+    ]
+
+
 def _static_value_provider_of(value_type):
   """"Helper function to plug a ValueProvider into argparse.
 
@@ -42,7 +56,7 @@ def _f(value):
   return _f
 
 
-class BeamArgumentParser(argparse.ArgumentParser):
+class _BeamArgumentParser(argparse.ArgumentParser):
   """An ArgumentParser that supports ValueProvider options.
 
   Example Usage::
@@ -133,7 +147,7 @@ def __init__(self, flags=None, **kwargs):
     """
     self._flags = flags
     self._all_options = kwargs
-    parser = BeamArgumentParser()
+    parser = _BeamArgumentParser()
 
     for cls in type(self).mro():
       if cls == PipelineOptions:
@@ -187,7 +201,7 @@ def get_all_options(self, drop_default=False):
     # TODO(BEAM-1319): PipelineOption sub-classes in the main session might be
     # repeated. Pick last unique instance of each subclass to avoid conflicts.
     subset = {}
-    parser = BeamArgumentParser()
+    parser = _BeamArgumentParser()
     for cls in PipelineOptions.__subclasses__():
       subset[str(cls)] = cls
     for cls in subset.values():
diff --git a/sdks/python/apache_beam/options/pipeline_options_validator.py b/sdks/python/apache_beam/options/pipeline_options_validator.py
index 5c1ce2ac9b1b..24d2e55eadbe 100644
--- a/sdks/python/apache_beam/options/pipeline_options_validator.py
+++ b/sdks/python/apache_beam/options/pipeline_options_validator.py
@@ -16,6 +16,8 @@
 #
 
 """Pipeline options validator.
+
+For internal use only; no backwards-compatibility guarantees.
 """
 import re
 
diff --git a/sdks/python/apache_beam/options/value_provider.py b/sdks/python/apache_beam/options/value_provider.py
index c00d7bc92bce..40bddba5a19c 100644
--- a/sdks/python/apache_beam/options/value_provider.py
+++ b/sdks/python/apache_beam/options/value_provider.py
@@ -24,6 +24,14 @@
 from apache_beam import error
 
 
+__all__ = [
+    'ValueProvider',
+    'StaticValueProvider',
+    'RuntimeValueProvider',
+    'check_accessible',
+    ]
+
+
 class ValueProvider(object):
   def is_accessible(self):
     raise NotImplementedError(

From 1b89697e5f92bbea4a5d5a8f838409d41dcfceb4 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 11 May 2017 13:30:32 -0700
Subject: [PATCH 201/387] [BEAM-1345] Mark Pipeline as public.

---
 sdks/python/apache_beam/pipeline.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index 83c72877c701..ec8dde4d1e75 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -67,6 +67,9 @@
 from apache_beam.utils.annotations import deprecated
 
 
+__all__ = ['Pipeline']
+
+
 class Pipeline(object):
   """A pipeline object that manages a DAG of PValues and their PTransforms.
 
@@ -182,6 +185,8 @@ def __exit__(self, exc_type, exc_val, exc_tb):
   def visit(self, visitor):
     """Visits depth-first every node of a pipeline's DAG.
 
+    Runner-internal implementation detail; no backwards-compatibility guarantees
+
     Args:
       visitor: PipelineVisitor object whose callbacks will be called for each
         node visited. See PipelineVisitor comments.
@@ -333,6 +338,7 @@ def visit_value(self, value, _):
     return Visitor.ok
 
   def to_runner_api(self):
+    """For internal use only; no backwards-compatibility guarantees."""
     from apache_beam.runners import pipeline_context
     from apache_beam.runners.api import beam_runner_api_pb2
     context = pipeline_context.PipelineContext()
@@ -346,6 +352,7 @@ def to_runner_api(self):
 
   @staticmethod
   def from_runner_api(proto, runner, options):
+    """For internal use only; no backwards-compatibility guarantees."""
     p = Pipeline(runner=runner, options=options)
     from apache_beam.runners import pipeline_context
     context = pipeline_context.PipelineContext(proto.components)

From c8317345ae469008f9ab371855e329d8b2eb32b7 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 11 May 2017 13:41:24 -0700
Subject: [PATCH 202/387] [BEAM-1345] Clearly delineate public api in runners
 package.

---
 sdks/python/apache_beam/runners/api/__init__.py          | 4 +++-
 sdks/python/apache_beam/runners/common.py                | 5 ++++-
 sdks/python/apache_beam/runners/dataflow/__init__.py     | 9 +++++++++
 .../apache_beam/runners/dataflow/dataflow_runner.py      | 3 +++
 .../apache_beam/runners/dataflow/test_dataflow_runner.py | 3 +++
 sdks/python/apache_beam/runners/direct/__init__.py       | 6 +++++-
 sdks/python/apache_beam/runners/direct/direct_runner.py  | 3 +++
 sdks/python/apache_beam/runners/pipeline_context.py      | 6 ++++++
 sdks/python/apache_beam/runners/portability/__init__.py  | 2 ++
 sdks/python/apache_beam/runners/runner.py                | 3 +++
 sdks/python/apache_beam/runners/worker/__init__.py       | 2 ++
 11 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/sdks/python/apache_beam/runners/api/__init__.py b/sdks/python/apache_beam/runners/api/__init__.py
index e94673cbc68d..bf95208bd6f9 100644
--- a/sdks/python/apache_beam/runners/api/__init__.py
+++ b/sdks/python/apache_beam/runners/api/__init__.py
@@ -15,7 +15,9 @@
 # limitations under the License.
 #
 
-"""Checked in to avoid protoc dependency for Python development.
+"""For internal use only; no backwards-compatibility guarantees.
+
+Checked in to avoid protoc dependency for Python development.
 
 Regenerate files with::
 
diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py
index 86db711378be..84535692806b 100644
--- a/sdks/python/apache_beam/runners/common.py
+++ b/sdks/python/apache_beam/runners/common.py
@@ -17,7 +17,10 @@
 
 # cython: profile=True
 
-"""Worker operations executor."""
+"""Worker operations executor.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 import sys
 import traceback
diff --git a/sdks/python/apache_beam/runners/dataflow/__init__.py b/sdks/python/apache_beam/runners/dataflow/__init__.py
index cce3acad34a4..6674ba5d9ff9 100644
--- a/sdks/python/apache_beam/runners/dataflow/__init__.py
+++ b/sdks/python/apache_beam/runners/dataflow/__init__.py
@@ -14,3 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+"""The DataflowRunner executes pipelines on Google Cloud Dataflow.
+
+Anything in this package not imported here is an internal implementation detail
+with no backwards-compatibility guarantees.
+"""
+
+from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
+from apache_beam.runners.dataflow.test_dataflow_runner import TestDataflowRunner
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 796a67b8cbdd..3d8437cf744d 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -46,6 +46,9 @@
 from apache_beam.options.pipeline_options import StandardOptions
 
 
+__all__ = ['DataflowRunner']
+
+
 class DataflowRunner(PipelineRunner):
   """A runner that creates job graphs and submits them for remote execution.
 
diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
index 4fd1026c8285..b339882de0f5 100644
--- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py
@@ -22,6 +22,9 @@
 from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner
 
 
+__all__ = ['TestDataflowRunner']
+
+
 class TestDataflowRunner(DataflowRunner):
   def run(self, pipeline):
     """Execute test pipeline and verify test matcher"""
diff --git a/sdks/python/apache_beam/runners/direct/__init__.py b/sdks/python/apache_beam/runners/direct/__init__.py
index 0d64513ff249..0f8275674bb5 100644
--- a/sdks/python/apache_beam/runners/direct/__init__.py
+++ b/sdks/python/apache_beam/runners/direct/__init__.py
@@ -15,5 +15,9 @@
 # limitations under the License.
 #
 
-"""Inprocess runner executes pipelines locally in a single process."""
+"""Inprocess runner executes pipelines locally in a single process.
+
+Anything in this package not imported here is an internal implementation detail
+with no backwards-compatibility guarantees.
+"""
 from apache_beam.runners.direct.direct_runner import DirectRunner
diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py
index 535aac33ed96..ecf5114feefd 100644
--- a/sdks/python/apache_beam/runners/direct/direct_runner.py
+++ b/sdks/python/apache_beam/runners/direct/direct_runner.py
@@ -36,6 +36,9 @@
 from apache_beam.options.value_provider import RuntimeValueProvider
 
 
+__all__ = ['DirectRunner']
+
+
 class DirectRunner(PipelineRunner):
   """Executes a single pipeline on the local machine."""
 
diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py
index d3d3c2468321..1c89d0652a26 100644
--- a/sdks/python/apache_beam/runners/pipeline_context.py
+++ b/sdks/python/apache_beam/runners/pipeline_context.py
@@ -15,6 +15,12 @@
 # limitations under the License.
 #
 
+"""Utility class for serializing pipelines via the runner API.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
+
+
 from apache_beam import pipeline
 from apache_beam import pvalue
 from apache_beam import coders
diff --git a/sdks/python/apache_beam/runners/portability/__init__.py b/sdks/python/apache_beam/runners/portability/__init__.py
index cce3acad34a4..7af93ed945fa 100644
--- a/sdks/python/apache_beam/runners/portability/__init__.py
+++ b/sdks/python/apache_beam/runners/portability/__init__.py
@@ -14,3 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+"""This runner is experimental; no backwards-compatibility guarantees."""
diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py
index d875fdc11a45..af00d8f27ee0 100644
--- a/sdks/python/apache_beam/runners/runner.py
+++ b/sdks/python/apache_beam/runners/runner.py
@@ -26,6 +26,9 @@
 import tempfile
 
 
+__all__ = ['PipelineRunner', 'PipelineState', 'PipelineResult']
+
+
 def _get_runner_map(runner_names, module_path):
   """Create a map of runner name in lower case to full import path to the
   runner class.
diff --git a/sdks/python/apache_beam/runners/worker/__init__.py b/sdks/python/apache_beam/runners/worker/__init__.py
index cce3acad34a4..0bce5d68f724 100644
--- a/sdks/python/apache_beam/runners/worker/__init__.py
+++ b/sdks/python/apache_beam/runners/worker/__init__.py
@@ -14,3 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+"""For internal use only; no backwards-compatibility guarantees."""

From 05557c85882be7ff21a91cf1ec42c056baf3cb1a Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@google.com>
Date: Wed, 10 May 2017 17:13:06 -0700
Subject: [PATCH 203/387] [BEAM-1345] Clearly delineate public api in
 apache_beam/coders.

---
 sdks/python/apache_beam/coders/coder_impl.py  |  2 ++
 sdks/python/apache_beam/coders/coders.py      | 32 ++++++++++++++++---
 sdks/python/apache_beam/coders/coders_test.py | 11 ++++---
 .../apache_beam/coders/coders_test_common.py  |  2 +-
 sdks/python/apache_beam/coders/observable.py  |  5 ++-
 sdks/python/apache_beam/coders/slow_stream.py |  5 ++-
 .../coders/standard_coders_test.py            |  2 +-
 sdks/python/apache_beam/coders/stream.pyx     |  5 +++
 sdks/python/apache_beam/coders/typecoders.py  |  3 ++
 .../examples/snippets/snippets_test.py        |  3 +-
 sdks/python/apache_beam/io/fileio_test.py     |  2 +-
 sdks/python/apache_beam/io/textio.py          |  2 +-
 sdks/python/apache_beam/transforms/window.py  |  2 +-
 13 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py
index a0496a2bbfb5..10298bfbf3b2 100644
--- a/sdks/python/apache_beam/coders/coder_impl.py
+++ b/sdks/python/apache_beam/coders/coder_impl.py
@@ -23,6 +23,8 @@
 
 This module may be optionally compiled with Cython, using the corresponding
 coder_impl.pxd file for type hints.
+
+For internal use only; no backwards-compatibility guarantees.
 """
 from types import NoneType
 
diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index 4f75182ab63d..ce914ddec0e0 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""Collection of useful coders."""
+"""Collection of useful coders.
+
+Only those coders listed in __all__ are part of the public API of this module.
+"""
 
 import base64
 import cPickle as pickle
@@ -45,6 +48,13 @@
   import dill
 
 
+__all__ = ['Coder',
+           'BytesCoder', 'DillCoder', 'FastPrimitivesCoder', 'FloatCoder',
+           'IterableCoder', 'PickleCoder', 'ProtoCoder', 'SingletonCoder',
+           'StrUtf8Coder', 'TimestampCoder', 'TupleCoder',
+           'TupleSequenceCoder', 'VarIntCoder', 'WindowedValueCoder']
+
+
 def serialize_coder(coder):
   from apache_beam.internal import pickler
   return '%s$%s' % (coder.__class__.__name__, pickler.dumps(coder))
@@ -116,6 +126,10 @@ def _create_impl(self):
                                         self.estimate_size)
 
   def get_impl(self):
+    """For internal use only; no backwards-compatibility guarantees.
+
+    Returns the CoderImpl backing this Coder.
+    """
     if not hasattr(self, '_impl'):
       self._impl = self._create_impl()
       assert isinstance(self._impl, coder_impl.CoderImpl)
@@ -152,13 +166,17 @@ def value_coder(self):
       raise ValueError('Not a KV coder: %s.' % self)
 
   def _get_component_coders(self):
-    """Returns the internal component coders of this coder."""
+    """For internal use only; no backwards-compatibility guarantees.
+
+    Returns the internal component coders of this coder."""
     # This is an internal detail of the Coder API and does not need to be
     # refined in user-defined Coders.
     return []
 
   def as_cloud_object(self):
-    """Returns Google Cloud Dataflow API description of this coder."""
+    """For internal use only; no backwards-compatibility guarantees.
+
+    Returns Google Cloud Dataflow API description of this coder."""
     # This is an internal detail of the Coder API and does not need to be
     # refined in user-defined Coders.
 
@@ -184,6 +202,8 @@ def __eq__(self, other):
     # pylint: enable=protected-access
 
   def to_runner_api(self, context):
+    """For internal use only; no backwards-compatibility guarantees.
+    """
     # TODO(BEAM-115): Use specialized URNs and components.
     from apache_beam.runners.api import beam_runner_api_pb2
     return beam_runner_api_pb2.Coder(
@@ -196,6 +216,8 @@ def to_runner_api(self, context):
 
   @staticmethod
   def from_runner_api(proto, context):
+    """For internal use only; no backwards-compatibility guarantees.
+    """
     any_proto = proto.spec.spec.parameter
     bytes_proto = google.protobuf.wrappers_pb2.BytesValue()
     any_proto.Unpack(bytes_proto)
@@ -779,7 +801,9 @@ def __hash__(self):
 
 
 class LengthPrefixCoder(FastCoder):
-  """Coder which prefixes the length of the encoded object in the stream."""
+  """For internal use only; no backwards-compatibility guarantees.
+
+  Coder which prefixes the length of the encoded object in the stream."""
 
   def __init__(self, value_coder):
     self._value_coder = value_coder
diff --git a/sdks/python/apache_beam/coders/coders_test.py b/sdks/python/apache_beam/coders/coders_test.py
index 575503b817a9..c89e81028b8a 100644
--- a/sdks/python/apache_beam/coders/coders_test.py
+++ b/sdks/python/apache_beam/coders/coders_test.py
@@ -20,8 +20,9 @@
 import logging
 import unittest
 
-from apache_beam import coders
+from apache_beam.coders import coders
 from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
+from apache_beam.coders.typecoders import registry as coders_registry
 
 
 class PickleCoderTest(unittest.TestCase):
@@ -46,13 +47,13 @@ def test_equality(self):
 class CodersTest(unittest.TestCase):
 
   def test_str_utf8_coder(self):
-    real_coder = coders.registry.get_coder(str)
+    real_coder = coders_registry.get_coder(str)
     expected_coder = coders.BytesCoder()
     self.assertEqual(
         real_coder.encode('abc'), expected_coder.encode('abc'))
     self.assertEqual('abc', real_coder.decode(real_coder.encode('abc')))
 
-    real_coder = coders.registry.get_coder(bytes)
+    real_coder = coders_registry.get_coder(bytes)
     expected_coder = coders.BytesCoder()
     self.assertEqual(
         real_coder.encode('abc'), expected_coder.encode('abc'))
@@ -82,7 +83,7 @@ def test_proto_coder(self):
     mb.field1 = True
     ma.field1 = u'hello world'
     expected_coder = coders.ProtoCoder(ma.__class__)
-    real_coder = coders.registry.get_coder(ma.__class__)
+    real_coder = coders_registry.get_coder(ma.__class__)
     self.assertEqual(expected_coder, real_coder)
     self.assertEqual(real_coder.encode(ma), expected_coder.encode(ma))
     self.assertEqual(ma, real_coder.decode(real_coder.encode(ma)))
@@ -104,7 +105,7 @@ class FallbackCoderTest(unittest.TestCase):
   def test_default_fallback_path(self):
     """Test fallback path picks a matching coder if no coder is registered."""
 
-    coder = coders.registry.get_coder(DummyClass)
+    coder = coders_registry.get_coder(DummyClass)
     # No matching coder, so picks the last fallback coder which is a
     # FastPrimitivesCoder.
     self.assertEqual(coder, coders.FastPrimitivesCoder())
diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py
index e5bfe35c4525..c9b67b346239 100644
--- a/sdks/python/apache_beam/coders/coders_test_common.py
+++ b/sdks/python/apache_beam/coders/coders_test_common.py
@@ -23,12 +23,12 @@
 
 import dill
 
-import coders
 import observable
 from apache_beam.transforms import window
 from apache_beam.utils import timestamp
 from apache_beam.utils import windowed_value
 
+from apache_beam.coders import coders
 from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
 
 
diff --git a/sdks/python/apache_beam/coders/observable.py b/sdks/python/apache_beam/coders/observable.py
index 5a808d8ad38c..fc952cf4e559 100644
--- a/sdks/python/apache_beam/coders/observable.py
+++ b/sdks/python/apache_beam/coders/observable.py
@@ -16,7 +16,10 @@
 #
 
 
-"""Observable base class for iterables."""
+"""Observable base class for iterables.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 
 class ObservableMixin(object):
diff --git a/sdks/python/apache_beam/coders/slow_stream.py b/sdks/python/apache_beam/coders/slow_stream.py
index 85837bcfa002..1ab55d90f98d 100644
--- a/sdks/python/apache_beam/coders/slow_stream.py
+++ b/sdks/python/apache_beam/coders/slow_stream.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""A pure Python implementation of stream.pyx."""
+"""A pure Python implementation of stream.pyx.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 import struct
 
diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py
index 885e88f6eea5..5f9845564477 100644
--- a/sdks/python/apache_beam/coders/standard_coders_test.py
+++ b/sdks/python/apache_beam/coders/standard_coders_test.py
@@ -26,7 +26,7 @@
 
 import yaml
 
-from apache_beam import coders
+from apache_beam.coders import coders
 from apache_beam.coders import coder_impl
 from apache_beam.utils import windowed_value
 from apache_beam.utils.timestamp import Timestamp
diff --git a/sdks/python/apache_beam/coders/stream.pyx b/sdks/python/apache_beam/coders/stream.pyx
index ae24418ca0bb..8d9768169220 100644
--- a/sdks/python/apache_beam/coders/stream.pyx
+++ b/sdks/python/apache_beam/coders/stream.pyx
@@ -15,6 +15,11 @@
 # limitations under the License.
 #
 
+"""Compiled version of the Stream objects used by CoderImpl.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
+
 cimport libc.stdlib
 cimport libc.string
 
diff --git a/sdks/python/apache_beam/coders/typecoders.py b/sdks/python/apache_beam/coders/typecoders.py
index 60832c9b130c..3894bb588172 100644
--- a/sdks/python/apache_beam/coders/typecoders.py
+++ b/sdks/python/apache_beam/coders/typecoders.py
@@ -71,6 +71,9 @@ def MakeXyzs(v):
 from apache_beam.typehints import typehints
 
 
+__all__ = ['registry']
+
+
 class CoderRegistry(object):
   """A coder registry for typehint/coder associations."""
 
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index 85d8bde186fb..211da245e33e 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -29,6 +29,7 @@
 from apache_beam import coders
 from apache_beam import pvalue
 from apache_beam import typehints
+from apache_beam.coders.coders import ToStringCoder
 from apache_beam.transforms.util import assert_that
 from apache_beam.transforms.util import equal_to
 from apache_beam.options.pipeline_options import PipelineOptions
@@ -422,7 +423,7 @@ class WriteDoFn(beam.DoFn):
       def __init__(self, file_to_write):
         self.file_to_write = file_to_write
         self.file_obj = None
-        self.coder = coders.ToStringCoder()
+        self.coder = ToStringCoder()
 
       def start_bundle(self):
         assert self.file_to_write
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 4c25505d5212..b92b8beda7ff 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -29,7 +29,7 @@
 import mock
 
 import apache_beam as beam
-from apache_beam import coders
+from apache_beam.coders import coders
 from apache_beam.io import fileio
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayData
diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py
index 750ec45b7e57..d43f4fca9ee7 100644
--- a/sdks/python/apache_beam/io/textio.py
+++ b/sdks/python/apache_beam/io/textio.py
@@ -21,7 +21,7 @@
 from __future__ import absolute_import
 import logging
 
-from apache_beam import coders
+from apache_beam.coders import coders
 from apache_beam.io import filebasedsource
 from apache_beam.io import fileio
 from apache_beam.io import iobase
diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py
index 29994c0b208a..6d0db3a987c1 100644
--- a/sdks/python/apache_beam/transforms/window.py
+++ b/sdks/python/apache_beam/transforms/window.py
@@ -53,7 +53,7 @@
 
 from google.protobuf import struct_pb2
 
-from apache_beam import coders
+from apache_beam.coders import coders
 from apache_beam.runners.api import beam_runner_api_pb2
 from apache_beam.transforms import timeutil
 from apache_beam.utils import proto_utils

From b8196295700629121b295ec31b1c8de0284c62c9 Mon Sep 17 00:00:00 2001
From: "chamikara@google.com" <chamikara@google.com>
Date: Thu, 11 May 2017 11:46:46 -0700
Subject: [PATCH 204/387] [BEAM-1340] Adds __all__ tags to classes in package
 apache_beam/io.

---
 sdks/python/apache_beam/io/concat_source.py              | 4 +++-
 sdks/python/apache_beam/io/filebasedsource.py            | 2 ++
 sdks/python/apache_beam/io/fileio.py                     | 7 +++++++
 sdks/python/apache_beam/io/filesystem.py                 | 3 +++
 sdks/python/apache_beam/io/filesystems.py                | 2 ++
 sdks/python/apache_beam/io/gcp/gcsfilesystem.py          | 2 ++
 sdks/python/apache_beam/io/gcp/gcsio.py                  | 3 +++
 sdks/python/apache_beam/io/gcp/pubsub.py                 | 2 ++
 sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py | 3 +++
 sdks/python/apache_beam/io/iobase.py                     | 2 ++
 sdks/python/apache_beam/io/localfilesystem.py            | 2 ++
 sdks/python/apache_beam/io/range_trackers.py             | 7 ++++++-
 sdks/python/apache_beam/io/source_test_utils.py          | 8 ++++++++
 13 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/io/concat_source.py b/sdks/python/apache_beam/io/concat_source.py
index dfd16953bfa9..56c4ccabcf71 100644
--- a/sdks/python/apache_beam/io/concat_source.py
+++ b/sdks/python/apache_beam/io/concat_source.py
@@ -15,7 +15,9 @@
 # limitations under the License.
 #
 
-"""Concat Source, which reads the union of several other sources.
+"""For internal use only; no backwards-compatibility guarantees.
+
+Concat Source, which reads the union of several other sources.
 """
 
 import bisect
diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py
index 215e01510224..bb9efc4b7d02 100644
--- a/sdks/python/apache_beam/io/filebasedsource.py
+++ b/sdks/python/apache_beam/io/filebasedsource.py
@@ -38,6 +38,8 @@
 
 MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 25
 
+__all__ = ['FileBasedSource']
+
 
 class FileBasedSource(iobase.BoundedSource):
   """A ``BoundedSource`` for reading a file glob of a given type."""
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index ca3a759797bb..aa1809318dad 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -37,6 +37,8 @@
 
 DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN'
 
+__all__ = ['FileBasedSink']
+
 
 class FileSink(iobase.Sink):
   """A sink to a GCS or local files.
@@ -280,6 +282,11 @@ def __eq__(self, other):
     return type(self) == type(other) and self.__dict__ == other.__dict__
 
 
+# Using FileBasedSink for the public API to be symmetric with FileBasedSource.
+# TODO: move code from FileSink to here and delete that class.
+FileBasedSink = FileSink
+
+
 class FileSinkWriter(iobase.Writer):
   """The writer for FileSink.
   """
diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py
index 3d35f3e49ce4..db6a1d0f6e6c 100644
--- a/sdks/python/apache_beam/io/filesystem.py
+++ b/sdks/python/apache_beam/io/filesystem.py
@@ -30,6 +30,9 @@
 
 DEFAULT_READ_BUFFER_SIZE = 16 * 1024 * 1024
 
+__all__ = ['CompressionTypes', 'CompressedFile', 'FileMetadata', 'FileSystem',
+           'MatchResult']
+
 
 class CompressionTypes(object):
   """Enum-like class representing known compression types."""
diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py
index 29f06442a9a1..e03968628d59 100644
--- a/sdks/python/apache_beam/io/filesystems.py
+++ b/sdks/python/apache_beam/io/filesystems.py
@@ -33,6 +33,8 @@
   pass
 # pylint: enable=wrong-import-position, unused-import
 
+__all__ = ['FileSystems']
+
 
 class FileSystems(object):
   """A class that defines the functions that can be performed on a filesystem.
diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
index dc71fcec6459..ce8b5e6e4244 100644
--- a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
+++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py
@@ -26,6 +26,8 @@
 from apache_beam.io.filesystem import MatchResult
 from apache_beam.io.gcp import gcsio
 
+__all__ = ['GCSFileSystem']
+
 
 class GCSFileSystem(FileSystem):
   """A GCS ``FileSystem`` implementation for accessing files on GCS.
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index c76c99d14c54..774ee549fc1f 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -34,6 +34,9 @@
 
 from apache_beam.utils import retry
 
+__all__ = ['GcsIO']
+
+
 # Issue a friendlier error message if the storage library is not available.
 # TODO(silviuc): Remove this guard when storage is available everywhere.
 try:
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index efc628d430a8..103fce0b8714 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -26,6 +26,8 @@
 from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
 from apache_beam.transforms.display import DisplayDataItem
 
+__all__ = ['PubSubSink', 'PubSubSource']
+
 
 class PubSubSource(dataflow_io.NativeSource):
   """Source for reading from a given Cloud Pub/Sub topic.
diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
index f42b70fc19f4..844cbc5fa224 100644
--- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
+++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py
@@ -24,6 +24,9 @@
 from apache_beam.testing.test_utils import compute_hash
 from apache_beam.utils import retry
 
+__all__ = ['BigqueryMatcher']
+
+
 # Protect against environments where bigquery library is not available.
 # pylint: disable=wrong-import-order, wrong-import-position
 try:
diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py
index e8ffb7270640..a80b12f72cfb 100644
--- a/sdks/python/apache_beam/io/iobase.py
+++ b/sdks/python/apache_beam/io/iobase.py
@@ -46,6 +46,8 @@
 from apache_beam.transforms.display import DisplayDataItem
 from apache_beam.utils.windowed_value import WindowedValue
 
+__all__ = ['BoundedSource', 'RangeTracker', 'Read', 'Sink', 'Write', 'Writer']
+
 
 # Encapsulates information about a bundle of a source generated when method
 # BoundedSource.split() is invoked.
diff --git a/sdks/python/apache_beam/io/localfilesystem.py b/sdks/python/apache_beam/io/localfilesystem.py
index c6707041b8e5..b08ac494846e 100644
--- a/sdks/python/apache_beam/io/localfilesystem.py
+++ b/sdks/python/apache_beam/io/localfilesystem.py
@@ -29,6 +29,8 @@
 from apache_beam.io.filesystem import FileSystem
 from apache_beam.io.filesystem import MatchResult
 
+__all__ = ['LocalFileSystem']
+
 
 class LocalFileSystem(FileSystem):
   """A Local ``FileSystem`` implementation for accessing files on disk.
diff --git a/sdks/python/apache_beam/io/range_trackers.py b/sdks/python/apache_beam/io/range_trackers.py
index 000df819dbe4..9cb36e73dc8e 100644
--- a/sdks/python/apache_beam/io/range_trackers.py
+++ b/sdks/python/apache_beam/io/range_trackers.py
@@ -24,6 +24,9 @@
 
 from apache_beam.io import iobase
 
+__all__ = ['OffsetRangeTracker', 'LexicographicKeyRangeTracker',
+           'OrderedPositionRangeTracker', 'UnsplittableRangeTracker']
+
 
 class OffsetRangeTracker(iobase.RangeTracker):
   """A 'RangeTracker' for non-negative positions of type 'long'."""
@@ -191,7 +194,9 @@ def set_split_points_unclaimed_callback(self, callback):
 
 
 class GroupedShuffleRangeTracker(iobase.RangeTracker):
-  """A 'RangeTracker' for positions used by'GroupedShuffleReader'.
+  """For internal use only; no backwards-compatibility guarantees.
+
+  A 'RangeTracker' for positions used by'GroupedShuffleReader'.
 
   These positions roughly correspond to hashes of keys. In case of hash
   collisions, multiple groups can have the same position. In that case, the
diff --git a/sdks/python/apache_beam/io/source_test_utils.py b/sdks/python/apache_beam/io/source_test_utils.py
index edb6409f4cc9..a144a8a8e5e4 100644
--- a/sdks/python/apache_beam/io/source_test_utils.py
+++ b/sdks/python/apache_beam/io/source_test_utils.py
@@ -52,6 +52,14 @@
 from multiprocessing.pool import ThreadPool
 from apache_beam.io import iobase
 
+__all__ = ['read_from_source', 'assert_sources_equal_reference_source',
+           'assert_reentrant_reads_succeed',
+           'assert_split_at_fraction_behavior',
+           'assert_split_at_fraction_binary',
+           'assert_split_at_fraction_exhaustive',
+           'assert_split_at_fraction_fails',
+           'assert_split_at_fraction_succeeds_and_consistent']
+
 
 class ExpectedSplitOutcome(object):
   MUST_SUCCEED_AND_BE_CONSISTENT = 1

From 101141da32e01c6873be7479eadb3f7efb998b54 Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Wed, 10 May 2017 23:20:20 -0700
Subject: [PATCH 205/387] Add __all__ tags to modules in package
 apache_beam/testing

---
 .../apache_beam/testing/pipeline_verifiers.py      |  8 ++++++++
 sdks/python/apache_beam/testing/test_pipeline.py   |  5 +++++
 sdks/python/apache_beam/testing/test_stream.py     | 14 +++++++++++++-
 sdks/python/apache_beam/testing/test_utils.py      |  6 +++++-
 4 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers.py b/sdks/python/apache_beam/testing/pipeline_verifiers.py
index 5a6082aaee50..a08eb5409cdd 100644
--- a/sdks/python/apache_beam/testing/pipeline_verifiers.py
+++ b/sdks/python/apache_beam/testing/pipeline_verifiers.py
@@ -32,6 +32,14 @@
 from apache_beam.testing import test_utils as utils
 from apache_beam.utils import retry
 
+
+__all__ = [
+    'PipelineStateMatcher',
+    'FileChecksumMatcher',
+    'retry_on_io_error_and_server_error',
+    ]
+
+
 try:
   from apitools.base.py.exceptions import HttpError
 except ImportError:
diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py
index 20f4839bdb2b..13b1639ea499 100644
--- a/sdks/python/apache_beam/testing/test_pipeline.py
+++ b/sdks/python/apache_beam/testing/test_pipeline.py
@@ -27,6 +27,11 @@
 from nose.plugins.skip import SkipTest
 
 
+__all__ = [
+    'TestPipeline',
+    ]
+
+
 class TestPipeline(Pipeline):
   """TestPipeline class is used inside of Beam tests that can be configured to
   run against pipeline runner.
diff --git a/sdks/python/apache_beam/testing/test_stream.py b/sdks/python/apache_beam/testing/test_stream.py
index 7ae27b74734e..a06bcd0795f2 100644
--- a/sdks/python/apache_beam/testing/test_stream.py
+++ b/sdks/python/apache_beam/testing/test_stream.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""Provides TestStream for verifying streaming runner semantics."""
+"""Provides TestStream for verifying streaming runner semantics.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 from abc import ABCMeta
 from abc import abstractmethod
@@ -28,6 +31,15 @@
 from apache_beam.utils.windowed_value import WindowedValue
 
 
+__all__ = [
+    'Event',
+    'ElementEvent',
+    'WatermarkEvent',
+    'ProcessingTimeEvent',
+    'TestStream',
+    ]
+
+
 class Event(object):
   """Test stream event to be emitted during execution of a TestStream."""
 
diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py
index 666207eb4d75..9feb80e144e6 100644
--- a/sdks/python/apache_beam/testing/test_utils.py
+++ b/sdks/python/apache_beam/testing/test_utils.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""Utility methods for testing"""
+"""Utility methods for testing
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 import hashlib
 import imp
@@ -23,6 +26,7 @@
 
 from apache_beam.utils import retry
 
+
 DEFAULT_HASHING_ALG = 'sha1'
 
 

From b5cc8b85f3fbe279d0817782b01caaee4c981a3c Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Thu, 11 May 2017 14:39:55 -0700
Subject: [PATCH 206/387] fix lint error in fake_datastore.py

---
 sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py b/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
index 0caf6d612333..23325793c15c 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
@@ -31,6 +31,7 @@
   pass
 # pylint: enable=wrong-import-order, wrong-import-position
 
+
 def create_run_query(entities, batch_size):
   """A fake datastore run_query method that returns entities in batches.
 

From dbb850c3c3dedc7f62d90cdf7f15fe7cff79ce1b Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 11 May 2017 09:26:30 -0700
Subject: [PATCH 207/387] Use a consistent calculation for GC Time

Truncate all garbage collection timestamps to be at the end of the
global window at the latest.

Add a reshuffle test, which was failing when late data arrived.

Update ReifyTimestamps to permit infinite skew. Elements that have
timestamps extracted from them may be late, but that is not the concern
of ReifyTimestamps.
---
 .../core/LateDataDroppingDoFnRunner.java      |  2 +-
 .../beam/runners/core/LateDataUtils.java      | 33 ++++++-
 .../beam/runners/core/ReduceFnRunner.java     | 43 +++------
 .../beam/runners/core/SimpleDoFnRunner.java   |  2 +-
 .../beam/runners/core/StatefulDoFnRunner.java |  6 +-
 .../beam/runners/core/WatermarkHold.java      |  9 +-
 .../beam/runners/core/LateDataUtilsTest.java  | 90 +++++++++++++++++++
 .../beam/sdk/transforms/ReifyTimestamps.java  |  6 ++
 .../apache/beam/sdk/transforms/Reshuffle.java | 29 +++---
 .../sdk/transforms/ReifyTimestampsTest.java   | 36 ++++++++
 .../beam/sdk/transforms/ReshuffleTest.java    | 27 ++++++
 11 files changed, 234 insertions(+), 49 deletions(-)
 create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataUtilsTest.java

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
index 66385c116ffb..570f52463666 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
@@ -159,7 +159,7 @@ public boolean apply(WindowedValue<InputT> input) {
     /** Is {@code window} expired w.r.t. the garbage collection watermark? */
     private boolean canDropDueToExpiredWindow(BoundedWindow window) {
       Instant inputWM = timerInternals.currentInputWatermarkTime();
-      return window.maxTimestamp().plus(windowingStrategy.getAllowedLateness()).isBefore(inputWM);
+      return LateDataUtils.garbageCollectionTime(window, windowingStrategy).isBefore(inputWM);
     }
   }
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
index 732e60c6c657..8a2b7c6508b0 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
@@ -24,15 +24,46 @@
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.metrics.CounterCell;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.WindowingStrategy;
-
+import org.joda.time.Duration;
+import org.joda.time.Instant;
 
 /**
  * Utils to handle late data.
  */
 public class LateDataUtils {
+  private LateDataUtils() {}
+
+  /**
+   * Return when {@code window} should be garbage collected. If the window's expiration time is on
+   * or after the end of the global window, it will be truncated to the end of the global window.
+   */
+  public static Instant garbageCollectionTime(
+      BoundedWindow window, WindowingStrategy windowingStrategy) {
+    return garbageCollectionTime(window, windowingStrategy.getAllowedLateness());
+  }
+
+  /**
+   * Return when {@code window} should be garbage collected. If the window's expiration time is on
+   * or after the end of the global window, it will be truncated to the end of the global window.
+   */
+  public static Instant garbageCollectionTime(BoundedWindow window, Duration allowedLateness) {
+
+    // If the end of the window + allowed lateness is beyond the "end of time" aka the end of the
+    // global window, then we truncate it. The conditional is phrased like it is because the
+    // addition of EOW + allowed lateness might even overflow the maximum allowed Instant
+    if (GlobalWindow.INSTANCE
+        .maxTimestamp()
+        .minus(allowedLateness)
+        .isBefore(window.maxTimestamp())) {
+      return GlobalWindow.INSTANCE.maxTimestamp();
+    } else {
+      return window.maxTimestamp().plus(allowedLateness);
+    }
+  }
 
   /**
    * Returns an {@code Iterable<WindowedValue<InputT>>} that only contains non-late input elements.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
index d2ed835ba463..62d519f6f8e5 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java
@@ -48,7 +48,6 @@
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
@@ -663,7 +662,7 @@ private class EnrichedTimerData {
       W window = directContext.window();
       this.isEndOfWindow = TimeDomain.EVENT_TIME == timer.getDomain()
           && timer.getTimestamp().equals(window.maxTimestamp());
-      Instant cleanupTime = garbageCollectionTime(window);
+      Instant cleanupTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy);
       this.isGarbageCollection = !timer.getTimestamp().isBefore(cleanupTime);
     }
 
@@ -769,9 +768,11 @@ public void onTimers(Iterable<TimerData> timers) throws Exception {
           // cleanup event and handled by the above).
           // Note we must do this even if the trigger is finished so that we are sure to cleanup
           // any final trigger finished bits.
-          checkState(windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO),
+          checkState(
+              windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO),
               "Unexpected zero getAllowedLateness");
-          Instant cleanupTime = garbageCollectionTime(directContext.window());
+          Instant cleanupTime =
+              LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy);
           WindowTracing.debug(
               "ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with "
                   + "inputWatermark:{}; outputWatermark:{}",
@@ -957,6 +958,7 @@ private Instant onTrigger(
     // Extract the window hold, and as a side effect clear it.
     final WatermarkHold.OldAndNewHolds pair =
         watermarkHold.extractAndRelease(renamedContext, isFinished).read();
+    // TODO: This isn't accurate if the elements are late. See BEAM-2262
     final Instant outputTimestamp = pair.oldHold;
     @Nullable Instant newHold = pair.newHold;
 
@@ -972,11 +974,12 @@ private Instant onTrigger(
       if (newHold.isAfter(directContext.window().maxTimestamp())) {
         // The hold must be for garbage collection, which can't have happened yet.
         checkState(
-          newHold.isEqual(garbageCollectionTime(directContext.window())),
-          "new hold %s should be at garbage collection for window %s plus %s",
-          newHold,
-          directContext.window(),
-          windowingStrategy.getAllowedLateness());
+            newHold.isEqual(
+                LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy)),
+            "new hold %s should be at garbage collection for window %s plus %s",
+            newHold,
+            directContext.window(),
+            windowingStrategy.getAllowedLateness());
       } else {
         // The hold must be for the end-of-window, which can't have happened yet.
         checkState(
@@ -1042,7 +1045,7 @@ private Instant scheduleEndOfWindowOrGarbageCollectionTimer(
     String which;
     Instant timer;
     if (endOfWindow.isBefore(inputWM)) {
-      timer = garbageCollectionTime(directContext.window());
+      timer = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy);
       which = "garbage collection";
     } else {
       timer = endOfWindow;
@@ -1072,28 +1075,10 @@ private void cancelEndOfWindowAndGarbageCollectionTimers(
         timerInternals.currentOutputWatermarkTime());
     Instant eow = directContext.window().maxTimestamp();
     directContext.timers().deleteTimer(eow, TimeDomain.EVENT_TIME);
-    Instant gc = garbageCollectionTime(directContext.window());
+    Instant gc = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy);
     if (gc.isAfter(eow)) {
       directContext.timers().deleteTimer(gc, TimeDomain.EVENT_TIME);
     }
   }
 
-  /**
-   * Return when {@code window} should be garbage collected. If the window's expiration time is on
-   * or after the end of the global window, it will be truncated to the end of the global window.
-   */
-  private Instant garbageCollectionTime(W window) {
-
-    // If the end of the window + allowed lateness is beyond the "end of time" aka the end of the
-    // global window, then we truncate it. The conditional is phrased like it is because the
-    // addition of EOW + allowed lateness might even overflow the maximum allowed Instant
-    if (GlobalWindow.INSTANCE
-        .maxTimestamp()
-        .minus(windowingStrategy.getAllowedLateness())
-        .isBefore(window.maxTimestamp())) {
-      return GlobalWindow.INSTANCE.maxTimestamp();
-    } else {
-      return window.maxTimestamp().plus(windowingStrategy.getAllowedLateness());
-    }
-  }
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 85423c0431b4..7ca305e51d0a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -949,7 +949,7 @@ public Timer align(Duration period) {
      */
     private Instant minTargetAndGcTime(Instant target) {
       if (TimeDomain.EVENT_TIME.equals(spec.getTimeDomain())) {
-        Instant windowExpiry = window.maxTimestamp().plus(allowedLateness);
+        Instant windowExpiry = LateDataUtils.garbageCollectionTime(window, allowedLateness);
         if (target.isAfter(windowExpiry)) {
           return windowExpiry;
         }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
index 28a9deef8795..c68a94319e9a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java
@@ -104,7 +104,7 @@ public void processElement(WindowedValue<InputT> input) {
   }
 
   private boolean isLate(BoundedWindow window) {
-    Instant gcTime = window.maxTimestamp().plus(windowingStrategy.getAllowedLateness());
+    Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy);
     Instant inputWM = cleanupTimer.currentInputWatermarkTime();
     return gcTime.isBefore(inputWM);
   }
@@ -208,7 +208,7 @@ public Instant currentInputWatermarkTime() {
 
     @Override
     public void setForWindow(BoundedWindow window) {
-      Instant gcTime = window.maxTimestamp().plus(windowingStrategy.getAllowedLateness());
+      Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy);
       // make sure this fires after any window.maxTimestamp() timers
       gcTime = gcTime.plus(GC_DELAY_MS);
       timerInternals.setTimer(StateNamespaces.window(windowCoder, window),
@@ -222,7 +222,7 @@ public boolean isForWindow(
         Instant timestamp,
         TimeDomain timeDomain) {
       boolean isEventTimer = timeDomain.equals(TimeDomain.EVENT_TIME);
-      Instant gcTime = window.maxTimestamp().plus(windowingStrategy.getAllowedLateness());
+      Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy);
       gcTime = gcTime.plus(GC_DELAY_MS);
       return isEventTimer && GC_TIMER_ID.equals(timerId) && gcTime.equals(timestamp);
     }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
index 64f5d9b165cc..13e4c43d66a6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java
@@ -365,8 +365,7 @@ private Instant addGarbageCollectionHold(
       ReduceFn<?, ?, ?, W>.Context context, boolean paneIsEmpty) {
     Instant outputWM = timerInternals.currentOutputWatermarkTime();
     Instant inputWM = timerInternals.currentInputWatermarkTime();
-    Instant eow = context.window().maxTimestamp();
-    Instant gcHold = eow.plus(windowingStrategy.getAllowedLateness());
+    Instant gcHold = LateDataUtils.garbageCollectionTime(context.window(), windowingStrategy);
 
     if (!windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO)) {
       WindowTracing.trace(
@@ -387,6 +386,12 @@ private Instant addGarbageCollectionHold(
       return null;
     }
 
+    if (!gcHold.isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
+      // If the garbage collection hold is past the timestamp we can represent, instead truncate
+      // to the maximum timestamp that is not positive infinity. This ensures all windows will
+      // eventually be garbage collected.
+      gcHold = BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.millis(1L));
+    }
     checkState(!gcHold.isBefore(inputWM),
         "Garbage collection hold %s cannot be before input watermark %s",
         gcHold, inputWM);
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataUtilsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataUtilsTest.java
new file mode 100644
index 000000000000..f0f315d39328
--- /dev/null
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataUtilsTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.joda.time.ReadableInstant;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link LateDataUtils}.
+ */
+@RunWith(JUnit4.class)
+public class LateDataUtilsTest {
+  @Test
+  public void beforeEndOfGlobalWindowSame() {
+    FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
+    Duration allowedLateness = Duration.standardMinutes(2);
+    WindowingStrategy<?, ?> strategy =
+        WindowingStrategy.globalDefault()
+            .withWindowFn(windowFn)
+            .withAllowedLateness(allowedLateness);
+
+    IntervalWindow window = windowFn.assignWindow(new Instant(10));
+    assertThat(
+        LateDataUtils.garbageCollectionTime(window, strategy),
+        equalTo(window.maxTimestamp().plus(allowedLateness)));
+  }
+
+  @Test
+  public void garbageCollectionTimeAfterEndOfGlobalWindow() {
+    FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
+    WindowingStrategy<?, ?> strategy =
+        WindowingStrategy.globalDefault()
+            .withWindowFn(windowFn);
+
+    IntervalWindow window = windowFn.assignWindow(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE));
+    assertThat(
+        window.maxTimestamp(),
+        Matchers.<ReadableInstant>greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
+    assertThat(
+        LateDataUtils.garbageCollectionTime(window, strategy),
+        equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
+  }
+
+  @Test
+  public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
+    FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
+    Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
+    WindowingStrategy<?, ?> strategy =
+        WindowingStrategy.globalDefault()
+            .withWindowFn(windowFn)
+            .withAllowedLateness(allowedLateness);
+
+    IntervalWindow window = windowFn.assignWindow(new Instant(-100));
+    assertThat(
+        window.maxTimestamp().plus(allowedLateness),
+        Matchers.<ReadableInstant>greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
+    assertThat(
+        LateDataUtils.garbageCollectionTime(window, strategy),
+        equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ReifyTimestamps.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ReifyTimestamps.java
index 0b1ab2588672..990f235456f4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ReifyTimestamps.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ReifyTimestamps.java
@@ -21,6 +21,7 @@
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
 
 /**
  * {@link PTransform PTransforms} for reifying the timestamp of values and reemitting the original
@@ -63,6 +64,11 @@ public void processElement(ProcessContext context) {
 
   private static class ExtractTimestampedValueDoFn<K, V>
       extends DoFn<KV<K, TimestampedValue<V>>, KV<K, V>> {
+    @Override
+    public Duration getAllowedTimestampSkew() {
+      return Duration.millis(Long.MAX_VALUE);
+    }
+
     @ProcessElement
     public void processElement(ProcessContext context) {
       KV<K, TimestampedValue<V>> kv = context.element();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
index 5394826e8551..3b7122c656f0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
@@ -71,22 +71,27 @@ public PCollection<KV<K, V>> expand(PCollection<KV<K, V>> input) {
             .withTimestampCombiner(TimestampCombiner.EARLIEST)
             .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
 
-    return input.apply(rewindow)
+    return input
+        .apply(rewindow)
         .apply("ReifyOriginalTimestamps", ReifyTimestamps.<K, V>inValues())
         .apply(GroupByKey.<K, TimestampedValue<V>>create())
         // Set the windowing strategy directly, so that it doesn't get counted as the user having
         // set allowed lateness.
         .setWindowingStrategyInternal(originalStrategy)
-        .apply("ExpandIterable", ParDo.of(
-            new DoFn<KV<K, Iterable<TimestampedValue<V>>>, KV<K, TimestampedValue<V>>>() {
-              @ProcessElement
-              public void processElement(ProcessContext c) {
-                K key = c.element().getKey();
-                for (TimestampedValue<V> value : c.element().getValue()) {
-                  c.output(KV.of(key, value));
-                }
-              }
-            }))
-        .apply("RestoreOriginalTimestamps", ReifyTimestamps.<K, V>extractFromValues());
+        .apply(
+            "ExpandIterable",
+            ParDo.of(
+                new DoFn<KV<K, Iterable<TimestampedValue<V>>>, KV<K, TimestampedValue<V>>>() {
+                  @ProcessElement
+                  public void processElement(ProcessContext c) {
+                    K key = c.element().getKey();
+                    for (TimestampedValue<V> value : c.element().getValue()) {
+                      c.output(KV.of(key, value));
+                    }
+                  }
+                }))
+        .apply(
+            "RestoreOriginalTimestamps",
+            ReifyTimestamps.<K, V>extractFromValues());
   }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReifyTimestampsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReifyTimestampsTest.java
index 181433eaafc1..e8728423743d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReifyTimestampsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReifyTimestampsTest.java
@@ -101,4 +101,40 @@ public void verifyTimestampsEqualValue(ProcessContext context) {
 
     pipeline.run();
   }
+
+  @Test
+  @Category(ValidatesRunner.class)
+  public void extractFromValuesWhenValueTimestampedLaterSucceeds() {
+    PCollection<KV<String, TimestampedValue<Integer>>> preified =
+        pipeline.apply(
+            Create.timestamped(
+                TimestampedValue.of(
+                    KV.of("foo", TimestampedValue.of(0, new Instant((0)))), new Instant(100)),
+                TimestampedValue.of(
+                    KV.of("foo", TimestampedValue.of(1, new Instant(1))), new Instant(101L)),
+                TimestampedValue.of(
+                    KV.of("bar", TimestampedValue.of(2, new Instant(2))), new Instant(102L)),
+                TimestampedValue.of(
+                    KV.of("baz", TimestampedValue.of(3, new Instant(3))), new Instant(103L))));
+
+    PCollection<KV<String, Integer>> timestamped =
+        preified.apply(ReifyTimestamps.<String, Integer>extractFromValues());
+
+    PAssert.that(timestamped)
+        .containsInAnyOrder(KV.of("foo", 0), KV.of("foo", 1), KV.of("bar", 2), KV.of("baz", 3));
+
+    timestamped.apply(
+        "AssertElementTimestamps",
+        ParDo.of(
+            new DoFn<KV<String, Integer>, Void>() {
+              @ProcessElement
+              public void verifyTimestampsEqualValue(ProcessContext context) {
+                assertThat(
+                    new Instant(context.element().getValue().longValue()),
+                    equalTo(context.timestamp()));
+              }
+            }));
+
+    pipeline.run();
+  }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
index 1038fd63810b..3cd7cf9cc69f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
@@ -27,8 +27,11 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestStream;
+import org.apache.beam.sdk.testing.UsesTestStream;
 import org.apache.beam.sdk.testing.ValidatesRunner;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
@@ -263,4 +266,28 @@ public void testReshuffleAfterSlidingWindows() {
 
     pipeline.run();
   }
+
+  @Test
+  @Category({ValidatesRunner.class, UsesTestStream.class})
+  public void testReshuffleWithTimestampsStreaming() {
+    TestStream<Long> stream =
+        TestStream.create(VarLongCoder.of())
+            .advanceWatermarkTo(new Instant(0L).plus(Duration.standardDays(48L)))
+            .addElements(
+                TimestampedValue.of(0L, new Instant(0L)),
+                TimestampedValue.of(1L, new Instant(0L).plus(Duration.standardDays(48L))),
+                TimestampedValue.of(
+                    2L, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(48L))))
+            .advanceWatermarkToInfinity();
+    PCollection<KV<String, Long>> input =
+        pipeline
+            .apply(stream).apply(WithKeys.<String, Long>of(""))
+            .apply(
+                Window.<KV<String, Long>>into(FixedWindows.of(Duration.standardMinutes(10L))));
+
+    PCollection<KV<String, Long>> reshuffled = input.apply(Reshuffle.<String, Long>of());
+    PAssert.that(reshuffled.apply(Values.<Long>create())).containsInAnyOrder(0L, 1L, 2L);
+
+    pipeline.run();
+  }
 }

From e10662df9d080162ab1c4f664165c5dcd1962c08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jean-Baptiste=20Onofr=C3=A9?= <jbonofre@apache.org>
Date: Thu, 11 May 2017 22:09:50 +0200
Subject: [PATCH 208/387] [BEAM-2256] Add the last previous range filter

---
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 47 ++++++++++++++-----
 .../beam/sdk/io/mongodb/MongoDbIOTest.java    | 18 +++++++
 2 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
index 7236a50655e5..620df74f2464 100644
--- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
+++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java
@@ -21,6 +21,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
 import com.mongodb.BasicDBObject;
 import com.mongodb.MongoClient;
 import com.mongodb.MongoClientURI;
@@ -184,7 +185,11 @@ public void populateDisplayData(DisplayData.Builder builder) {
     }
   }
 
-  private static class BoundedMongoDbSource extends BoundedSource<Document> {
+  /**
+   * A MongoDB {@link BoundedSource} reading {@link Document} from  a given instance.
+   */
+  @VisibleForTesting
+  static class BoundedMongoDbSource extends BoundedSource<Document> {
     private Read spec;
 
     private BoundedMongoDbSource(Read spec) {
@@ -294,7 +299,8 @@ public List<BoundedSource<Document>> split(long desiredBundleSizeBytes,
      * @param additionalFilter A custom (user) additional filter to append to the range filters.
      * @return A list of filters containing the ranges.
      */
-    private static List<String> splitKeysToFilters(List<Document> splitKeys, String
+    @VisibleForTesting
+    static List<String> splitKeysToFilters(List<Document> splitKeys, String
         additionalFilter) {
       ArrayList<String> filters = new ArrayList<>();
       String lowestBound = null; // lower boundary (previous split in the iteration)
@@ -306,30 +312,45 @@ private static List<String> splitKeysToFilters(List<Document> splitKeys, String
           // the range from the beginning up to this split
           rangeFilter = String.format("{ $and: [ {\"_id\":{$lte:ObjectId(\"%s\")}}",
               splitKey);
+          filters.add(formatFilter(rangeFilter, additionalFilter));
         } else if (i == splitKeys.size() - 1) {
-          // this is the last split in the list, the filter defines
-          // the range from the split up to the end
+          // this is the last split in the list, the filters define
+          // the range from the previous split to the current split and also
+          // the current split to the end
+          rangeFilter = String.format("{ $and: [ {\"_id\":{$gt:ObjectId(\"%s\"),"
+              + "$lte:ObjectId(\"%s\")}}", lowestBound, splitKey);
+          filters.add(formatFilter(rangeFilter, additionalFilter));
           rangeFilter = String.format("{ $and: [ {\"_id\":{$gt:ObjectId(\"%s\")}}",
               splitKey);
+          filters.add(formatFilter(rangeFilter, additionalFilter));
         } else {
           // we are between two splits
           rangeFilter = String.format("{ $and: [ {\"_id\":{$gt:ObjectId(\"%s\"),"
               + "$lte:ObjectId(\"%s\")}}", lowestBound, splitKey);
+          filters.add(formatFilter(rangeFilter, additionalFilter));
         }
-        if (additionalFilter != null && !additionalFilter.isEmpty()) {
-          // user provided a filter, we append the user filter to the range filter
-          rangeFilter = String.format("%s,%s ]}", rangeFilter, additionalFilter);
-        } else {
-          // user didn't provide a filter, just cleany close the range filter
-          rangeFilter = String.format("%s ]}", rangeFilter);
-        }
-
-        filters.add(rangeFilter);
 
         lowestBound = splitKey;
       }
       return filters;
     }
+
+    /**
+     * Cleanly format range filter, optionally adding the users filter if specified.
+     *
+     * @param filter The range filter.
+     * @param additionalFilter The users filter. Null if unspecified.
+     * @return The cleanly formatted range filter.
+     */
+    private static String formatFilter(String filter, @Nullable String additionalFilter) {
+      if (additionalFilter != null && !additionalFilter.isEmpty()) {
+        // user provided a filter, we append the user filter to the range filter
+        return String.format("%s,%s ]}", filter, additionalFilter);
+      } else {
+        // user didn't provide a filter, just cleanly close the range filter
+        return String.format("%s ]}", filter);
+      }
+    }
   }
 
   private static class BoundedMongoDbReader extends BoundedSource.BoundedReader<Document> {
diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
index 454c6ba1ea7b..cd26b483cda9 100644
--- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
+++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java
@@ -38,6 +38,8 @@
 import java.io.Serializable;
 import java.net.ServerSocket;
 import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Count;
@@ -138,6 +140,22 @@ public void stop() throws Exception {
     mongodExecutable.stop();
   }
 
+  @Test
+  public void testSplitIntoFilters() throws Exception {
+    ArrayList<Document> documents = new ArrayList<>();
+    documents.add(new Document("_id", 56));
+    documents.add(new Document("_id", 109));
+    documents.add(new Document("_id", 256));
+    List<String> filters = MongoDbIO.BoundedMongoDbSource.splitKeysToFilters(documents, null);
+    assertEquals(4, filters.size());
+    assertEquals("{ $and: [ {\"_id\":{$lte:ObjectId(\"56\")}} ]}", filters.get(0));
+    assertEquals("{ $and: [ {\"_id\":{$gt:ObjectId(\"56\"),$lte:ObjectId(\"109\")}} ]}",
+        filters.get(1));
+    assertEquals("{ $and: [ {\"_id\":{$gt:ObjectId(\"109\"),$lte:ObjectId(\"256\")}} ]}",
+        filters.get(2));
+    assertEquals("{ $and: [ {\"_id\":{$gt:ObjectId(\"256\")}} ]}", filters.get(3));
+  }
+
   @Test
   public void testFullRead() throws Exception {
 

From 2e6451dfea2a67b102cf0c334eb99608fd95a0b2 Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Wed, 10 May 2017 23:06:36 -0700
Subject: [PATCH 209/387] [BEAM-1340] Add __all__ tags to modules in package
 apache_beam/transforms

---
 .../examples/snippets/snippets_test.py        |  2 +-
 .../runners/dataflow/dataflow_runner.py       |  2 +-
 .../runners/dataflow/dataflow_runner_test.py  |  5 +++--
 sdks/python/apache_beam/transforms/core.py    | 20 +++++++++++++++++++
 .../apache_beam/transforms/cy_combiners.py    |  5 ++++-
 .../apache_beam/transforms/ptransform.py      |  9 +++++++++
 .../apache_beam/transforms/ptransform_test.py | 11 +++++-----
 .../apache_beam/transforms/sideinputs.py      |  2 ++
 .../python/apache_beam/transforms/timeutil.py |  5 +++++
 sdks/python/apache_beam/transforms/trigger.py | 14 +++++++++++++
 sdks/python/apache_beam/transforms/window.py  | 15 ++++++++++++++
 11 files changed, 80 insertions(+), 10 deletions(-)

diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index 211da245e33e..37cd4701efdd 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -892,7 +892,7 @@ def process(self, element):
         unix_timestamp = extract_timestamp_from_log_entry(element)
         # Wrap and emit the current entry and new timestamp in a
         # TimestampedValue.
-        yield beam.TimestampedValue(element, unix_timestamp)
+        yield beam.window.TimestampedValue(element, unix_timestamp)
 
     timestamped_items = items | 'timestamp' >> beam.ParDo(AddTimestampDoFn())
     # [END setting_timestamp]
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 3d8437cf744d..da8de9d1dce0 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -169,7 +169,7 @@ class GroupByKeyInputVisitor(PipelineVisitor):
       def visit_transform(self, transform_node):
         # Imported here to avoid circular dependencies.
         # pylint: disable=wrong-import-order, wrong-import-position
-        from apache_beam import GroupByKey, GroupByKeyOnly
+        from apache_beam.transforms.core import GroupByKey, GroupByKeyOnly
         if isinstance(transform_node.transform, (GroupByKey, GroupByKeyOnly)):
           pcoll = transform_node.inputs[0]
           input_type = pcoll.element_type
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
index b61a683f9689..ac9b0282fbcc 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
@@ -37,6 +37,7 @@
 from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.transforms.core import GroupByKeyOnly
 from apache_beam.typehints import typehints
 
 # Protect against environments where apitools library is not available.
@@ -184,7 +185,7 @@ def test_group_by_key_input_visitor_with_valid_inputs(self):
     pcoll1 = PCollection(p)
     pcoll2 = PCollection(p)
     pcoll3 = PCollection(p)
-    for transform in [beam.GroupByKeyOnly(), beam.GroupByKey()]:
+    for transform in [GroupByKeyOnly(), beam.GroupByKey()]:
       pcoll1.element_type = None
       pcoll2.element_type = typehints.Any
       pcoll3.element_type = typehints.KV[typehints.Any, typehints.Any]
@@ -198,7 +199,7 @@ def test_group_by_key_input_visitor_with_invalid_inputs(self):
     p = TestPipeline()
     pcoll1 = PCollection(p)
     pcoll2 = PCollection(p)
-    for transform in [beam.GroupByKeyOnly(), beam.GroupByKey()]:
+    for transform in [GroupByKeyOnly(), beam.GroupByKey()]:
       pcoll1.element_type = typehints.TupleSequenceConstraint
       pcoll2.element_type = typehints.Set
       err_msg = "Input to GroupByKey must be of Tuple or Any type"
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index d42115c5c23f..a1964cfd8e7f 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -53,6 +53,26 @@
 from apache_beam.options.pipeline_options import TypeOptions
 
 
+__all__ = [
+    'DoFn',
+    'CombineFn',
+    'PartitionFn',
+    'ParDo',
+    'FlatMap',
+    'Map',
+    'Filter',
+    'CombineGlobally',
+    'CombinePerKey',
+    'CombineValues',
+    'GroupByKey',
+    'Partition',
+    'Windowing',
+    'WindowInto',
+    'Flatten',
+    'Create',
+    ]
+
+
 # Type variables
 T = typehints.TypeVariable('T')
 K = typehints.TypeVariable('K')
diff --git a/sdks/python/apache_beam/transforms/cy_combiners.py b/sdks/python/apache_beam/transforms/cy_combiners.py
index f82487077ec7..84aee212790c 100644
--- a/sdks/python/apache_beam/transforms/cy_combiners.py
+++ b/sdks/python/apache_beam/transforms/cy_combiners.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""A library of basic cythonized CombineFn subclasses."""
+"""A library of basic cythonized CombineFn subclasses.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 from __future__ import absolute_import
 
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index fb79b19be982..d1f9835b307d 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -60,6 +60,13 @@ class and wrapper class that allows lambda functions to be used as
 from apache_beam.utils import urns
 
 
+__all__ = [
+    'PTransform',
+    'ptransform_fn',
+    'label_from_callable',
+    ]
+
+
 class _PValueishTransform(object):
   """Visitor for PValueish objects.
 
@@ -639,6 +646,8 @@ def default_label(self):
 def ptransform_fn(fn):
   """A decorator for a function-based PTransform.
 
+  Experimental; no backwards-compatibility guarantees.
+
   Args:
     fn: A function implementing a custom PTransform.
 
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index 137992dd428b..3320d79bbbb0 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -34,6 +34,7 @@
 import apache_beam.pvalue as pvalue
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms import window
+from apache_beam.transforms.core import GroupByKeyOnly
 import apache_beam.transforms.combiners as combine
 from apache_beam.transforms.display import DisplayData, DisplayDataItem
 from apache_beam.transforms.ptransform import PTransform
@@ -579,7 +580,7 @@ def test_group_by_key_only_input_must_be_kv_pairs(self):
     pipeline = TestPipeline()
     pcolls = pipeline | 'A' >> beam.Create(['a', 'b', 'f'])
     with self.assertRaises(typehints.TypeCheckError) as cm:
-      pcolls | 'D' >> beam.GroupByKeyOnly()
+      pcolls | 'D' >> GroupByKeyOnly()
       pipeline.run()
 
     expected_error_prefix = ('Input type hint violation at D: expected '
@@ -1087,7 +1088,7 @@ def test_group_by_key_only_output_type_deduction(self):
          | 'Str' >> beam.Create(['t', 'e', 's', 't']).with_output_types(str)
          | ('Pair' >> beam.Map(lambda x: (x, ord(x)))
             .with_output_types(typehints.KV[str, str]))
-         | beam.GroupByKeyOnly())
+         | GroupByKeyOnly())
 
     # Output type should correctly be deduced.
     # GBK-only should deduce that KV[A, B] is turned into KV[A, Iterable[B]].
@@ -1111,7 +1112,7 @@ def test_group_by_key_only_does_not_type_check(self):
     with self.assertRaises(typehints.TypeCheckError) as e:
       (self.p
        | beam.Create([1, 2, 3]).with_output_types(int)
-       | 'F' >> beam.GroupByKeyOnly())
+       | 'F' >> GroupByKeyOnly())
 
     self.assertEqual("Input type hint violation at F: "
                      "expected Tuple[TypeVariable[K], TypeVariable[V]], "
@@ -1154,7 +1155,7 @@ def test_pipeline_checking_gbk_insufficient_type_information(self):
       (self.p
        | 'Nums' >> beam.Create(range(5)).with_output_types(int)
        | 'ModDup' >> beam.Map(lambda x: (x % 2, x))
-       | beam.GroupByKeyOnly())
+       | GroupByKeyOnly())
 
     self.assertEqual('Pipeline type checking is enabled, however no output '
                      'type-hint was found for the PTransform '
@@ -1977,7 +1978,7 @@ def test_pardo_type_inference(self):
   def test_gbk_type_inference(self):
     self.assertEqual(
         typehints.Tuple[str, typehints.Iterable[int]],
-        beam.core.GroupByKeyOnly().infer_output_type(typehints.KV[str, int]))
+        GroupByKeyOnly().infer_output_type(typehints.KV[str, int]))
 
   def test_pipeline_inference(self):
     created = self.p | beam.Create(['a', 'b', 'c'])
diff --git a/sdks/python/apache_beam/transforms/sideinputs.py b/sdks/python/apache_beam/transforms/sideinputs.py
index 6ba53113a9df..f10cb92ed5e3 100644
--- a/sdks/python/apache_beam/transforms/sideinputs.py
+++ b/sdks/python/apache_beam/transforms/sideinputs.py
@@ -17,6 +17,8 @@
 
 """Internal side input transforms and implementations.
 
+For internal use only; no backwards-compatibility guarantees.
+
 Important: this module is an implementation detail and should not be used
 directly by pipeline writers. Instead, users should use the helper methods
 AsSingleton, AsIter, AsList and AsDict in apache_beam.pvalue.
diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py
index ba4ef36d548a..c0f91981d7d6 100644
--- a/sdks/python/apache_beam/transforms/timeutil.py
+++ b/sdks/python/apache_beam/transforms/timeutil.py
@@ -23,6 +23,11 @@
 from abc import abstractmethod
 
 
+__all__ = [
+    'TimeDomain',
+    ]
+
+
 class TimeDomain(object):
   """Time domain for streaming timers."""
 
diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py
index 2cb7ce32702d..7de2f85100a8 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -38,6 +38,20 @@
 from apache_beam.utils.timestamp import MIN_TIMESTAMP
 
 
+__all__ = [
+    'AccumulationMode',
+    'TriggerFn',
+    'DefaultTrigger',
+    'AfterWatermark',
+    'AfterCount',
+    'Repeatedly',
+    'AfterAny',
+    'AfterAll',
+    'AfterEach',
+    'OrFinally',
+    ]
+
+
 class AccumulationMode(object):
   """Controls what to do with data when a trigger fires multiple times.
   """
diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py
index 6d0db3a987c1..94187e0c5d54 100644
--- a/sdks/python/apache_beam/transforms/window.py
+++ b/sdks/python/apache_beam/transforms/window.py
@@ -65,6 +65,21 @@
 from apache_beam.utils.windowed_value import WindowedValue
 
 
+__all__ = [
+    'TimestampCombiner',
+    'WindowFn',
+    'BoundedWindow',
+    'IntervalWindow',
+    'TimestampedValue',
+    'GlobalWindow',
+    'NonMergingWindowFn',
+    'GlobalWindows',
+    'FixedWindows',
+    'SlidingWindows',
+    'Sessions',
+    ]
+
+
 # TODO(ccy): revisit naming and semantics once Java Apache Beam finalizes their
 # behavior.
 class TimestampCombiner(object):

From 4c28f054e7d5d3ed1bdf372ce6dc0096c9ce56a6 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Thu, 11 May 2017 14:50:33 -0700
Subject: [PATCH 210/387] Add internal comments to metrics

---
 sdks/python/apache_beam/metrics/__init__.py   | 1 +
 sdks/python/apache_beam/metrics/cells.py      | 2 ++
 sdks/python/apache_beam/metrics/execution.py  | 6 ++----
 sdks/python/apache_beam/metrics/metric.py     | 4 ++++
 sdks/python/apache_beam/metrics/metricbase.py | 2 ++
 5 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/metrics/__init__.py b/sdks/python/apache_beam/metrics/__init__.py
index 164d1a859815..8ce7bbb173fd 100644
--- a/sdks/python/apache_beam/metrics/__init__.py
+++ b/sdks/python/apache_beam/metrics/__init__.py
@@ -15,3 +15,4 @@
 # limitations under the License.
 #
 from apache_beam.metrics.metric import Metrics
+from apache_beam.metrics.metric import MetricsFilter
diff --git a/sdks/python/apache_beam/metrics/cells.py b/sdks/python/apache_beam/metrics/cells.py
index fbe3ad3efcd5..ba840f752b18 100644
--- a/sdks/python/apache_beam/metrics/cells.py
+++ b/sdks/python/apache_beam/metrics/cells.py
@@ -29,6 +29,8 @@
 from apache_beam.metrics.metricbase import Counter
 from apache_beam.metrics.metricbase import Distribution
 
+__all__ = ['DistributionResult']
+
 
 class CellCommitState(object):
   """For internal use only; no backwards-compatibility guarantees.
diff --git a/sdks/python/apache_beam/metrics/execution.py b/sdks/python/apache_beam/metrics/execution.py
index a06ec0c2f6af..675e49c1234c 100644
--- a/sdks/python/apache_beam/metrics/execution.py
+++ b/sdks/python/apache_beam/metrics/execution.py
@@ -24,7 +24,7 @@
 
 - MetricKey - Internal key for a metric.
 - MetricResult - Current status of a metric's updates/commits.
-- MetricsEnvironment - Keeps track of MetricsContainer and other metrics
+- _MetricsEnvironment - Keeps track of MetricsContainer and other metrics
     information for every single execution working thread.
 - MetricsContainer - Holds the metrics of a single step and a single
     unit-of-commit (bundle).
@@ -36,9 +36,7 @@
 
 
 class MetricKey(object):
-  """
-
-  Key used to identify instance of metric cell.
+  """Key used to identify instance of metric cell.
 
   Metrics are internally keyed by the step name they associated with and
   the name of the metric.
diff --git a/sdks/python/apache_beam/metrics/metric.py b/sdks/python/apache_beam/metrics/metric.py
index 33db4e111052..f99c0c415d56 100644
--- a/sdks/python/apache_beam/metrics/metric.py
+++ b/sdks/python/apache_beam/metrics/metric.py
@@ -30,6 +30,8 @@
 from apache_beam.metrics.metricbase import Counter, Distribution
 from apache_beam.metrics.metricbase import MetricName
 
+__all__ = ['Metrics', 'MetricsFilter']
+
 
 class Metrics(object):
   """Lets users create/access metric objects during pipeline execution."""
@@ -146,6 +148,8 @@ def query(self, filter=None):
 class MetricsFilter(object):
   """Simple object to filter metrics results.
 
+  This class is experimental. No backwards-compatibility guarantees.
+
   If filters by matching a result's step-namespace-name with three internal
   sets. No execution/matching logic is added to this object, so that it may
   be used to construct arguments as an RPC request. It is left for runners
diff --git a/sdks/python/apache_beam/metrics/metricbase.py b/sdks/python/apache_beam/metrics/metricbase.py
index fa0ca757d85a..699f29cb58ab 100644
--- a/sdks/python/apache_beam/metrics/metricbase.py
+++ b/sdks/python/apache_beam/metrics/metricbase.py
@@ -30,6 +30,8 @@
 - MetricName - Namespace and name used to refer to a Metric.
 """
 
+__all__ = ['Metric', 'Counter', 'Distribution', 'MetricName']
+
 
 class MetricName(object):
   """The name of a metric.

From d531219fcaac19d483d108c8e7df4476034c25f7 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 11 May 2017 12:07:00 -0700
Subject: [PATCH 211/387] [BEAM-1345] Annotate public members of pvalue.

---
 sdks/python/apache_beam/pvalue.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py
index fa91fe30de02..7385e82c3a5c 100644
--- a/sdks/python/apache_beam/pvalue.py
+++ b/sdks/python/apache_beam/pvalue.py
@@ -31,6 +31,17 @@
 from apache_beam import typehints
 
 
+__all__ = [
+    'PCollection',
+    'TaggedOutput',
+    'AsSingleton',
+    'AsIter',
+    'AsList',
+    'AsDict',
+    'EmptySideInput',
+]
+
+
 class PValue(object):
   """Base class for PCollection.
 

From 16a3fff18fdd34ebd85fe7d436f8ce3a429666d0 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 11 May 2017 12:09:28 -0700
Subject: [PATCH 212/387] [BEAM-1345] Mark apache_beam/internal as internal.

---
 sdks/python/apache_beam/internal/__init__.py     | 2 ++
 sdks/python/apache_beam/internal/gcp/__init__.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/sdks/python/apache_beam/internal/__init__.py b/sdks/python/apache_beam/internal/__init__.py
index cce3acad34a4..0bce5d68f724 100644
--- a/sdks/python/apache_beam/internal/__init__.py
+++ b/sdks/python/apache_beam/internal/__init__.py
@@ -14,3 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+"""For internal use only; no backwards-compatibility guarantees."""
diff --git a/sdks/python/apache_beam/internal/gcp/__init__.py b/sdks/python/apache_beam/internal/gcp/__init__.py
index cce3acad34a4..0bce5d68f724 100644
--- a/sdks/python/apache_beam/internal/gcp/__init__.py
+++ b/sdks/python/apache_beam/internal/gcp/__init__.py
@@ -14,3 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+"""For internal use only; no backwards-compatibility guarantees."""

From 7881fd5e6b53f6a26e23f877e47a410f6a515ae7 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 11 May 2017 12:54:13 -0700
Subject: [PATCH 213/387] [BEAM-1345] Clearly delineate public api in
 apache_beam/typehints.

---
 sdks/python/apache_beam/pipeline.py           |  2 +-
 sdks/python/apache_beam/transforms/core.py    |  8 ++++----
 .../apache_beam/transforms/ptransform.py      | 14 +++++++-------
 .../apache_beam/typehints/decorators.py       | 17 +++++++++++++----
 sdks/python/apache_beam/typehints/opcodes.py  |  2 ++
 .../typehints/trivial_inference.py            |  2 ++
 .../python/apache_beam/typehints/typecheck.py | 15 +++++++++------
 .../python/apache_beam/typehints/typehints.py | 18 ++++++++++++++++++
 .../apache_beam/typehints/typehints_test.py   | 19 ++++++++++---------
 9 files changed, 66 insertions(+), 31 deletions(-)

diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index ec8dde4d1e75..79480d77f194 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -53,11 +53,11 @@
 import tempfile
 
 from apache_beam import pvalue
-from apache_beam import typehints
 from apache_beam.internal import pickler
 from apache_beam.runners import create_runner
 from apache_beam.runners import PipelineRunner
 from apache_beam.transforms import ptransform
+from apache_beam.typehints import typehints
 from apache_beam.typehints import TypeCheckError
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.options.pipeline_options import SetupOptions
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index a1964cfd8e7f..abe699ff1607 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -40,15 +40,15 @@
 from apache_beam.transforms.window import GlobalWindows
 from apache_beam.transforms.window import WindowFn
 from apache_beam.typehints import Any
-from apache_beam.typehints import get_type_hints
-from apache_beam.typehints import is_consistent_with
 from apache_beam.typehints import Iterable
 from apache_beam.typehints import KV
 from apache_beam.typehints import trivial_inference
-from apache_beam.typehints import TypeCheckError
 from apache_beam.typehints import Union
-from apache_beam.typehints import WithTypeHints
+from apache_beam.typehints.decorators import get_type_hints
+from apache_beam.typehints.decorators import TypeCheckError
+from apache_beam.typehints.decorators import WithTypeHints
 from apache_beam.typehints.trivial_inference import element_type
+from apache_beam.typehints.typehints import is_consistent_with
 from apache_beam.utils import urns
 from apache_beam.options.pipeline_options import TypeOptions
 
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index d1f9835b307d..8898c36253fc 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -46,16 +46,16 @@ class and wrapper class that allows lambda functions to be used as
 
 from apache_beam import error
 from apache_beam import pvalue
-from apache_beam import typehints
 from apache_beam.internal import pickler
 from apache_beam.internal import util
 from apache_beam.transforms.display import HasDisplayData
 from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.typehints import getcallargs_forhints
-from apache_beam.typehints import TypeCheckError
-from apache_beam.typehints import validate_composite_type_param
-from apache_beam.typehints import WithTypeHints
+from apache_beam.typehints import typehints
+from apache_beam.typehints.decorators import getcallargs_forhints
+from apache_beam.typehints.decorators import TypeCheckError
+from apache_beam.typehints.decorators import WithTypeHints
 from apache_beam.typehints.trivial_inference import instance_to_type
+from apache_beam.typehints.typehints import validate_composite_type_param
 from apache_beam.utils import proto_utils
 from apache_beam.utils import urns
 
@@ -491,7 +491,7 @@ class PTransformWithSideInputs(PTransform):
   """
 
   def __init__(self, fn, *args, **kwargs):
-    if isinstance(fn, type) and issubclass(fn, typehints.WithTypeHints):
+    if isinstance(fn, type) and issubclass(fn, WithTypeHints):
       # Don't treat Fn class objects as callables.
       raise ValueError('Use %s() not %s.' % (fn.__name__, fn.__name__))
     self.fn = self.make_fn(fn)
@@ -577,7 +577,7 @@ def element_type(side_input):
           continue
         if not typehints.is_consistent_with(
             bindings.get(arg, typehints.Any), hint):
-          raise typehints.TypeCheckError(
+          raise TypeCheckError(
               'Type hint violation for \'%s\': requires %s but got %s for %s'
               % (self.label, hint, bindings[arg], arg))
 
diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py
index 4eabdba543c7..6ed388a85217 100644
--- a/sdks/python/apache_beam/typehints/decorators.py
+++ b/sdks/python/apache_beam/typehints/decorators.py
@@ -86,11 +86,20 @@ def foo((a, b)):
 import inspect
 import types
 
-from apache_beam.typehints import check_constraint
-from apache_beam.typehints import CompositeTypeHintError
-from apache_beam.typehints import SimpleTypeHintError
 from apache_beam.typehints import typehints
-from apache_beam.typehints import validate_composite_type_param
+from apache_beam.typehints.typehints import check_constraint
+from apache_beam.typehints.typehints import CompositeTypeHintError
+from apache_beam.typehints.typehints import SimpleTypeHintError
+from apache_beam.typehints.typehints import validate_composite_type_param
+
+
+__all__ = [
+    'with_input_types',
+    'with_output_types',
+    'WithTypeHints',
+    'TypeCheckError',
+]
+
 
 # This is missing in the builtin types module.  str.upper is arbitrary, any
 # method on a C-implemented type will do.
diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py
index 042acc0ca15b..83f444c6c53b 100644
--- a/sdks/python/apache_beam/typehints/opcodes.py
+++ b/sdks/python/apache_beam/typehints/opcodes.py
@@ -23,6 +23,8 @@
 
 Bytecodes with more complicated behavior (e.g. modifying the program counter)
 are handled inline rather than here.
+
+For internal use only; no backwards-compatibility guarantees.
 """
 import types
 
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py
index 4581aa1d31ac..977ea066a9f4 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -16,6 +16,8 @@
 #
 
 """Trivial type inference for simple functions.
+
+For internal use only; no backwards-compatibility guarantees.
 """
 import __builtin__
 import collections
diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py
index 160d10415f4e..09b73f994ddd 100644
--- a/sdks/python/apache_beam/typehints/typecheck.py
+++ b/sdks/python/apache_beam/typehints/typecheck.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""Runtime type checking support."""
+"""Runtime type checking support.
+
+For internal use only; no backwards-compatibility guarantees.
+"""
 
 import collections
 import inspect
@@ -25,13 +28,13 @@
 from apache_beam.pvalue import TaggedOutput
 from apache_beam.transforms.core import DoFn
 from apache_beam.transforms.window import WindowedValue
-from apache_beam.typehints import check_constraint
-from apache_beam.typehints import CompositeTypeHintError
-from apache_beam.typehints import GeneratorWrapper
-from apache_beam.typehints import SimpleTypeHintError
-from apache_beam.typehints import TypeCheckError
 from apache_beam.typehints.decorators import _check_instance_type
 from apache_beam.typehints.decorators import getcallargs_forhints
+from apache_beam.typehints.decorators import GeneratorWrapper
+from apache_beam.typehints.decorators import TypeCheckError
+from apache_beam.typehints.typehints import check_constraint
+from apache_beam.typehints.typehints import CompositeTypeHintError
+from apache_beam.typehints.typehints import SimpleTypeHintError
 
 
 class AbstractDoFnWrapper(DoFn):
diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py
index 9b41adbc115d..cc430be4a00e 100644
--- a/sdks/python/apache_beam/typehints/typehints.py
+++ b/sdks/python/apache_beam/typehints/typehints.py
@@ -68,6 +68,24 @@
 import types
 
 
+__all__ = [
+    'Any',
+    'Union',
+    'Optional',
+    'Tuple',
+    'Tuple',
+    'List',
+    'KV',
+    'Dict',
+    'Set',
+    'Iterable',
+    'Iterator',
+    'Generator',
+    'WindowedValue',
+    'TypeVariable',
+]
+
+
 # A set of the built-in Python types we don't support, guiding the users
 # to templated (upper-case) versions instead.
 DISALLOWED_PRIMITIVE_TYPES = (list, set, tuple, dict)
diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py
index f90b5e93b884..f1b92e096264 100644
--- a/sdks/python/apache_beam/typehints/typehints_test.py
+++ b/sdks/python/apache_beam/typehints/typehints_test.py
@@ -21,9 +21,8 @@
 import unittest
 
 
-import apache_beam.typehints as typehints
+import apache_beam.typehints.typehints as typehints
 from apache_beam.typehints import Any
-from apache_beam.typehints import is_consistent_with
 from apache_beam.typehints import Tuple
 from apache_beam.typehints import TypeCheckError
 from apache_beam.typehints import Union
@@ -34,6 +33,8 @@
 from apache_beam.typehints.decorators import _positional_arg_hints
 from apache_beam.typehints.decorators import get_type_hints
 from apache_beam.typehints.decorators import getcallargs_forhints
+from apache_beam.typehints.decorators import GeneratorWrapper
+from apache_beam.typehints.typehints import is_consistent_with
 
 
 def check_or_interleave(hint, value, var):
@@ -712,7 +713,7 @@ def count(n):
 
     l = []
     interleave_func = lambda x: l.append(x)
-    wrapped_gen = typehints.GeneratorWrapper(count(4), interleave_func)
+    wrapped_gen = GeneratorWrapper(count(4), interleave_func)
 
     # Should function as a normal generator.
     self.assertEqual(0, next(wrapped_gen))
@@ -1032,12 +1033,12 @@ def expand_ints(it):
 class DecoratorHelpers(TypeHintTestCase):
 
   def test_hint_helper(self):
-    self.assertTrue(typehints.is_consistent_with(Any, int))
-    self.assertTrue(typehints.is_consistent_with(int, Any))
-    self.assertTrue(typehints.is_consistent_with(str, object))
-    self.assertFalse(typehints.is_consistent_with(object, str))
-    self.assertTrue(typehints.is_consistent_with(str, Union[str, int]))
-    self.assertFalse(typehints.is_consistent_with(Union[str, int], str))
+    self.assertTrue(is_consistent_with(Any, int))
+    self.assertTrue(is_consistent_with(int, Any))
+    self.assertTrue(is_consistent_with(str, object))
+    self.assertFalse(is_consistent_with(object, str))
+    self.assertTrue(is_consistent_with(str, Union[str, int]))
+    self.assertFalse(is_consistent_with(Union[str, int], str))
 
   def test_positional_arg_hints(self):
     self.assertEquals(typehints.Any, _positional_arg_hints('x', {}))

From 0b7356e329f060c3824991cbe3b43bce89042763 Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Thu, 11 May 2017 15:07:30 -0700
Subject: [PATCH 214/387] Move assert_that, equal_to, is_empty to
 apache_beam.testing.util

---
 .../examples/complete/autocomplete_test.py    |   4 +-
 .../examples/complete/estimate_pi_test.py     |   4 +-
 .../complete/game/hourly_team_score_test.py   |   4 +-
 .../examples/complete/game/user_score_test.py |   4 +-
 .../examples/complete/tfidf_test.py           |   4 +-
 .../complete/top_wikipedia_sessions_test.py   |   4 +-
 .../cookbook/bigquery_side_input_test.py      |   4 +-
 .../cookbook/bigquery_tornadoes_test.py       |   6 +-
 .../examples/cookbook/coders_test.py          |   4 +-
 .../examples/cookbook/combiners_test.py       |   6 +-
 .../cookbook/custom_ptransform_test.py        |   4 +-
 .../examples/cookbook/filters_test.py         |  12 +-
 .../examples/cookbook/mergecontacts.py        |  14 ++-
 .../apache_beam/examples/snippets/snippets.py |  17 +--
 .../examples/snippets/snippets_test.py        |  30 ++---
 .../examples/wordcount_debugging.py           |   6 +-
 sdks/python/apache_beam/io/avroio_test.py     |   4 +-
 .../apache_beam/io/concat_source_test.py      |   4 +-
 .../apache_beam/io/filebasedsource_test.py    |   4 +-
 sdks/python/apache_beam/io/sources_test.py    |   4 +-
 sdks/python/apache_beam/io/textio_test.py     |   5 +-
 sdks/python/apache_beam/io/tfrecordio_test.py |  24 ++--
 sdks/python/apache_beam/pipeline_test.py      |   4 +-
 .../maptask_executor_runner_test.py           |   6 +-
 .../python/apache_beam/runners/runner_test.py |   4 +-
 sdks/python/apache_beam/testing/util.py       | 107 ++++++++++++++++++
 .../{transforms => testing}/util_test.py      |   4 +-
 .../apache_beam/transforms/combiners_test.py  |   2 +-
 .../apache_beam/transforms/create_test.py     |   3 +-
 .../apache_beam/transforms/ptransform_test.py |   2 +-
 .../apache_beam/transforms/sideinputs_test.py |   2 +-
 .../apache_beam/transforms/trigger_test.py    |   2 +-
 sdks/python/apache_beam/transforms/util.py    |  79 -------------
 .../apache_beam/transforms/window_test.py     |   2 +-
 .../transforms/write_ptransform_test.py       |   2 +-
 .../typehints/typed_pipeline_test.py          |   2 +-
 36 files changed, 223 insertions(+), 170 deletions(-)
 create mode 100644 sdks/python/apache_beam/testing/util.py
 rename sdks/python/apache_beam/{transforms => testing}/util_test.py (93%)

diff --git a/sdks/python/apache_beam/examples/complete/autocomplete_test.py b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
index 438633a25690..378d222bfa8d 100644
--- a/sdks/python/apache_beam/examples/complete/autocomplete_test.py
+++ b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
@@ -22,8 +22,8 @@
 import apache_beam as beam
 from apache_beam.examples.complete import autocomplete
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class AutocompleteTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
index 12d83794fefe..fd5130966887 100644
--- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
+++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
@@ -22,8 +22,8 @@
 
 from apache_beam.examples.complete import estimate_pi
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import BeamAssertException
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import BeamAssertException
 
 
 def in_between(lower, upper):
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py
index bd0abcaa1a2b..9c30127d7416 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py
@@ -23,6 +23,8 @@
 import apache_beam as beam
 from apache_beam.examples.complete.game import hourly_team_score
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class HourlyTeamScoreTest(unittest.TestCase):
@@ -44,7 +46,7 @@ def test_hourly_team_score(self):
                     start_min='2015-11-16-15-20',
                     stop_min='2015-11-16-17-20',
                     window_duration=60))
-      beam.assert_that(result, beam.equal_to([
+      assert_that(result, equal_to([
           ('team1', 18), ('team2', 2), ('team3', 13)]))
 
 
diff --git a/sdks/python/apache_beam/examples/complete/game/user_score_test.py b/sdks/python/apache_beam/examples/complete/game/user_score_test.py
index 2db53bdfb837..59903d985460 100644
--- a/sdks/python/apache_beam/examples/complete/game/user_score_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/user_score_test.py
@@ -23,6 +23,8 @@
 import apache_beam as beam
 from apache_beam.examples.complete.game import user_score
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class UserScoreTest(unittest.TestCase):
@@ -40,7 +42,7 @@ def test_user_score(self):
     with TestPipeline() as p:
       result = (
           p | beam.Create(UserScoreTest.SAMPLE_DATA) | user_score.UserScore())
-      beam.assert_that(result, beam.equal_to([
+      assert_that(result, equal_to([
           ('user1_team1', 50), ('user2_team2', 2), ('user3_team3', 8),
           ('user4_team3', 5)]))
 
diff --git a/sdks/python/apache_beam/examples/complete/tfidf_test.py b/sdks/python/apache_beam/examples/complete/tfidf_test.py
index 0e3025459445..f177dfc9e166 100644
--- a/sdks/python/apache_beam/examples/complete/tfidf_test.py
+++ b/sdks/python/apache_beam/examples/complete/tfidf_test.py
@@ -26,6 +26,8 @@
 import apache_beam as beam
 from apache_beam.examples.complete import tfidf
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 EXPECTED_RESULTS = set([
@@ -57,7 +59,7 @@ def test_tfidf_transform(self):
         uri_to_line
         | tfidf.TfIdf()
         | beam.Map(lambda (word, (uri, tfidf)): (word, uri, tfidf)))
-    beam.assert_that(result, beam.equal_to(EXPECTED_RESULTS))
+    assert_that(result, equal_to(EXPECTED_RESULTS))
     # Run the pipeline. Note that the assert_that above adds to the pipeline
     # a check that the result PCollection contains expected values. To actually
     # trigger the check the pipeline must be run.
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
index 4850c04cee98..5fb6276cb8d5 100644
--- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
+++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
@@ -24,6 +24,8 @@
 import apache_beam as beam
 from apache_beam.examples.complete import top_wikipedia_sessions
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class ComputeTopSessionsTest(unittest.TestCase):
@@ -54,7 +56,7 @@ def test_compute_top_sessions(self):
     edits = p | beam.Create(self.EDITS)
     result = edits | top_wikipedia_sessions.ComputeTopSessions(1.0)
 
-    beam.assert_that(result, beam.equal_to(self.EXPECTED))
+    assert_that(result, equal_to(self.EXPECTED))
     p.run()
 
 
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
index 1ca25c9e14e9..b11dc47c3363 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
@@ -23,6 +23,8 @@
 import apache_beam as beam
 from apache_beam.examples.cookbook import bigquery_side_input
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class BigQuerySideInputTest(unittest.TestCase):
@@ -42,7 +44,7 @@ def test_create_groups(self):
                                                words_pcoll, ignore_corpus_pcoll,
                                                ignore_word_pcoll)
 
-    beam.assert_that(groups, beam.equal_to(
+    assert_that(groups, equal_to(
         [('A', 'corpus2', 'word2'),
          ('B', 'corpus2', 'word2'),
          ('C', 'corpus2', 'word2')]))
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
index ca7ca9e2e74d..c926df8ec15c 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
@@ -23,6 +23,8 @@
 import apache_beam as beam
 from apache_beam.examples.cookbook import bigquery_tornadoes
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class BigQueryTornadoesTest(unittest.TestCase):
@@ -35,8 +37,8 @@ def test_basics(self):
         {'month': 1, 'day': 3, 'tornado': True},
         {'month': 2, 'day': 1, 'tornado': True}]))
     results = bigquery_tornadoes.count_tornadoes(rows)
-    beam.assert_that(results, beam.equal_to([{'month': 1, 'tornado_count': 2},
-                                             {'month': 2, 'tornado_count': 1}]))
+    assert_that(results, equal_to([{'month': 1, 'tornado_count': 2},
+                                   {'month': 2, 'tornado_count': 1}]))
     p.run().wait_until_finish()
 
 
diff --git a/sdks/python/apache_beam/examples/cookbook/coders_test.py b/sdks/python/apache_beam/examples/cookbook/coders_test.py
index 35cf252575d4..f71dad8ed80f 100644
--- a/sdks/python/apache_beam/examples/cookbook/coders_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/coders_test.py
@@ -23,8 +23,8 @@
 import apache_beam as beam
 from apache_beam.examples.cookbook import coders
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class CodersTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/cookbook/combiners_test.py b/sdks/python/apache_beam/examples/cookbook/combiners_test.py
index 45c779fd5159..ee1fb7765d47 100644
--- a/sdks/python/apache_beam/examples/cookbook/combiners_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/combiners_test.py
@@ -28,6 +28,8 @@
 
 import apache_beam as beam
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class CombinersTest(unittest.TestCase):
@@ -49,7 +51,7 @@ def test_combine_per_key_with_callable(self):
         | beam.Create(CombinersTest.SAMPLE_DATA)
         | beam.CombinePerKey(sum))
 
-    beam.assert_that(result, beam.equal_to([('a', 6), ('b', 30), ('c', 100)]))
+    assert_that(result, equal_to([('a', 6), ('b', 30), ('c', 100)]))
     result.pipeline.run()
 
   def test_combine_per_key_with_custom_callable(self):
@@ -65,7 +67,7 @@ def multiply(values):
         | beam.Create(CombinersTest.SAMPLE_DATA)
         | beam.CombinePerKey(multiply))
 
-    beam.assert_that(result, beam.equal_to([('a', 6), ('b', 200), ('c', 100)]))
+    assert_that(result, equal_to([('a', 6), ('b', 200), ('c', 100)]))
     result.pipeline.run()
 
 
diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
index 2d35d8d29d2a..c7c6dbabd5e0 100644
--- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
@@ -23,8 +23,8 @@
 import apache_beam as beam
 from apache_beam.examples.cookbook import custom_ptransform
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class CustomCountTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/examples/cookbook/filters_test.py b/sdks/python/apache_beam/examples/cookbook/filters_test.py
index 44a352f19e64..fd49f938742c 100644
--- a/sdks/python/apache_beam/examples/cookbook/filters_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/filters_test.py
@@ -23,6 +23,8 @@
 import apache_beam as beam
 from apache_beam.examples.cookbook import filters
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class FiltersTest(unittest.TestCase):
@@ -45,22 +47,22 @@ def _get_result_for_month(self, month):
   def test_basic(self):
     """Test that the correct result is returned for a simple dataset."""
     results = self._get_result_for_month(1)
-    beam.assert_that(
+    assert_that(
         results,
-        beam.equal_to([{'year': 2010, 'month': 1, 'day': 1, 'mean_temp': 3},
-                       {'year': 2012, 'month': 1, 'day': 2, 'mean_temp': 3}]))
+        equal_to([{'year': 2010, 'month': 1, 'day': 1, 'mean_temp': 3},
+                  {'year': 2012, 'month': 1, 'day': 2, 'mean_temp': 3}]))
     results.pipeline.run()
 
   def test_basic_empty(self):
     """Test that the correct empty result is returned for a simple dataset."""
     results = self._get_result_for_month(3)
-    beam.assert_that(results, beam.equal_to([]))
+    assert_that(results, equal_to([]))
     results.pipeline.run()
 
   def test_basic_empty_missing(self):
     """Test that the correct empty result is returned for a missing month."""
     results = self._get_result_for_month(4)
-    beam.assert_that(results, beam.equal_to([]))
+    assert_that(results, equal_to([]))
     results.pipeline.run()
 
 
diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
index 5aaba1015aa9..4f53c615ba97 100644
--- a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
+++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
@@ -40,6 +40,8 @@
 from apache_beam.io import WriteToText
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 def run(argv=None, assert_results=None):
@@ -118,12 +120,12 @@ def read_kv_textfile(label, textfile):
   # TODO(silviuc): Move the assert_results logic to the unit test.
   if assert_results is not None:
     expected_luddites, expected_writers, expected_nomads = assert_results
-    beam.assert_that(num_luddites, beam.equal_to([expected_luddites]),
-                     label='assert:luddites')
-    beam.assert_that(num_writers, beam.equal_to([expected_writers]),
-                     label='assert:writers')
-    beam.assert_that(num_nomads, beam.equal_to([expected_nomads]),
-                     label='assert:nomads')
+    assert_that(num_luddites, equal_to([expected_luddites]),
+                label='assert:luddites')
+    assert_that(num_writers, equal_to([expected_writers]),
+                label='assert:writers')
+    assert_that(num_nomads, equal_to([expected_nomads]),
+                label='assert:nomads')
   # Execute pipeline.
   return p.run()
 
diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py
index 1bdb9a366bf0..72595723de45 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets.py
@@ -33,6 +33,8 @@
 import apache_beam as beam
 from apache_beam.metrics import Metrics
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 # Quiet some pylint warnings that happen because of the somewhat special
 # format for the code snippets.
@@ -566,8 +568,9 @@ def process(self, element):
       | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
 
   # [START example_wordcount_debugging_assert]
-  beam.assert_that(
-      filtered_words, beam.equal_to([('Flourish', 3), ('stomach', 1)]))
+  beam.testing.util.assert_that(
+      filtered_words, beam.testing.util.equal_to(
+          [('Flourish', 3), ('stomach', 1)]))
   # [END example_wordcount_debugging_assert]
 
   output = (filtered_words
@@ -661,8 +664,8 @@ def model_custom_source(count):
   # [END model_custom_source_use_new_source]
 
   lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
-  beam.assert_that(
-      lines, beam.equal_to(
+  assert_that(
+      lines, equal_to(
           ['line ' + str(number) for number in range(0, count)]))
 
   p.run().wait_until_finish()
@@ -691,8 +694,8 @@ def expand(self, pcoll):
   # [END model_custom_source_use_ptransform]
 
   lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
-  beam.assert_that(
-      lines, beam.equal_to(
+  assert_that(
+      lines, equal_to(
           ['line ' + str(number) for number in range(0, count)]))
 
   # Don't test runner api due to pickling errors.
@@ -872,7 +875,7 @@ def model_textio_compressed(renames, expected):
       compression_type=beam.io.filesystem.CompressionTypes.GZIP)
   # [END model_textio_write_compressed]
 
-  beam.assert_that(lines, beam.equal_to(expected))
+  assert_that(lines, equal_to(expected))
   p.visit(SnippetUtils.RenameFiles(renames))
   p.run().wait_until_finish()
 
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index 37cd4701efdd..f7b51a75e7eb 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -30,10 +30,10 @@
 from apache_beam import pvalue
 from apache_beam import typehints
 from apache_beam.coders.coders import ToStringCoder
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.examples.snippets import snippets
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 from apache_beam.utils.windowed_value import WindowedValue
 
 # pylint: disable=expression-not-assigned
@@ -158,11 +158,11 @@ def filter_using_length(word, lower_bound, upper_bound=float('inf')):
                                                     avg_word_len))
     # [END model_pardo_side_input]
 
-    beam.assert_that(small_words, beam.equal_to(['a', 'bb', 'ccc']))
-    beam.assert_that(larger_than_average, beam.equal_to(['ccc', 'dddd']),
-                     label='larger_than_average')
-    beam.assert_that(small_but_nontrivial, beam.equal_to(['bb']),
-                     label='small_but_not_trivial')
+    assert_that(small_words, equal_to(['a', 'bb', 'ccc']))
+    assert_that(larger_than_average, equal_to(['ccc', 'dddd']),
+                label='larger_than_average')
+    assert_that(small_but_nontrivial, equal_to(['bb']),
+                label='small_but_not_trivial')
     p.run()
 
   def test_pardo_side_input_dofn(self):
@@ -816,7 +816,7 @@ def test_setting_fixed_windows(self):
               | 'group' >> beam.GroupByKey()
               | 'combine' >> beam.CombineValues(sum))
     unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    beam.assert_that(unkeyed, beam.equal_to([110, 215, 120]))
+    assert_that(unkeyed, equal_to([110, 215, 120]))
     p.run()
 
   def test_setting_sliding_windows(self):
@@ -834,8 +834,8 @@ def test_setting_sliding_windows(self):
               | 'group' >> beam.GroupByKey()
               | 'combine' >> beam.CombineValues(sum))
     unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    beam.assert_that(unkeyed,
-                     beam.equal_to([2, 2, 2, 18, 23, 39, 39, 39, 41, 41]))
+    assert_that(unkeyed,
+                equal_to([2, 2, 2, 18, 23, 39, 39, 39, 41, 41]))
     p.run()
 
   def test_setting_session_windows(self):
@@ -853,8 +853,8 @@ def test_setting_session_windows(self):
               | 'group' >> beam.GroupByKey()
               | 'combine' >> beam.CombineValues(sum))
     unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    beam.assert_that(unkeyed,
-                     beam.equal_to([29, 27]))
+    assert_that(unkeyed,
+                equal_to([29, 27]))
     p.run()
 
   def test_setting_global_window(self):
@@ -872,7 +872,7 @@ def test_setting_global_window(self):
               | 'group' >> beam.GroupByKey()
               | 'combine' >> beam.CombineValues(sum))
     unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    beam.assert_that(unkeyed, beam.equal_to([56]))
+    assert_that(unkeyed, equal_to([56]))
     p.run()
 
   def test_setting_timestamp(self):
@@ -903,7 +903,7 @@ def process(self, element):
               | 'group' >> beam.GroupByKey()
               | 'combine' >> beam.CombineValues(sum))
     unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    beam.assert_that(unkeyed, beam.equal_to([42, 187]))
+    assert_that(unkeyed, equal_to([42, 187]))
     p.run()
 
 
@@ -921,7 +921,7 @@ def expand(self, pcoll):
 
     p = TestPipeline()
     lengths = p | beam.Create(["a", "ab", "abc"]) | ComputeWordLengths()
-    beam.assert_that(lengths, beam.equal_to([1, 2, 3]))
+    assert_that(lengths, equal_to([1, 2, 3]))
     p.run()
 
 
diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py
index 98acde4a4752..ca9f7b6371de 100644
--- a/sdks/python/apache_beam/examples/wordcount_debugging.py
+++ b/sdks/python/apache_beam/examples/wordcount_debugging.py
@@ -51,6 +51,8 @@
 from apache_beam.metrics import Metrics
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.options.pipeline_options import SetupOptions
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class FilterTextFn(beam.DoFn):
@@ -133,8 +135,8 @@ def run(argv=None):
   # of the Pipeline implies that the expectations were  met. Learn more at
   # https://cloud.google.com/dataflow/pipelines/testing-your-pipeline on how to
   # test your pipeline.
-  beam.assert_that(
-      filtered_words, beam.equal_to([('Flourish', 3), ('stomach', 1)]))
+  assert_that(
+      filtered_words, equal_to([('Flourish', 3), ('stomach', 1)]))
 
   # Format the counts into a PCollection of strings and write the output using a
   # "Write" transform that has side effects.
diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py
index 4a2183928b98..6dcf121f6fc4 100644
--- a/sdks/python/apache_beam/io/avroio_test.py
+++ b/sdks/python/apache_beam/io/avroio_test.py
@@ -27,10 +27,10 @@
 from apache_beam.io import filebasedsource
 from apache_beam.io import source_test_utils
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
 
 # Importing following private class for testing purposes.
 from apache_beam.io.avroio import _AvroSource as AvroSource
diff --git a/sdks/python/apache_beam/io/concat_source_test.py b/sdks/python/apache_beam/io/concat_source_test.py
index a02f9ad64663..4a8f5193dee2 100644
--- a/sdks/python/apache_beam/io/concat_source_test.py
+++ b/sdks/python/apache_beam/io/concat_source_test.py
@@ -27,8 +27,8 @@
 from apache_beam.io import source_test_utils
 from apache_beam.io.concat_source import ConcatSource
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class RangeSource(iobase.BoundedSource):
diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py
index e17a004a3430..afb340d7608b 100644
--- a/sdks/python/apache_beam/io/filebasedsource_test.py
+++ b/sdks/python/apache_beam/io/filebasedsource_test.py
@@ -41,10 +41,10 @@
 from apache_beam.options.value_provider import StaticValueProvider
 from apache_beam.options.value_provider import RuntimeValueProvider
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
 
 
 class LineSource(FileBasedSource):
diff --git a/sdks/python/apache_beam/io/sources_test.py b/sdks/python/apache_beam/io/sources_test.py
index c0b8ad620dc8..10d401b8d2a2 100644
--- a/sdks/python/apache_beam/io/sources_test.py
+++ b/sdks/python/apache_beam/io/sources_test.py
@@ -28,8 +28,8 @@
 from apache_beam.io import iobase
 from apache_beam.io import range_trackers
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 class LineSource(iobase.BoundedSource):
diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py
index d00afef6c033..9a4ec47e583f 100644
--- a/sdks/python/apache_beam/io/textio_test.py
+++ b/sdks/python/apache_beam/io/textio_test.py
@@ -44,9 +44,8 @@
 from apache_beam.io.filesystem import CompressionTypes
 
 from apache_beam.testing.test_pipeline import TestPipeline
-
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 
 
 # TODO: Refactor code so all io tests are using same library
diff --git a/sdks/python/apache_beam/io/tfrecordio_test.py b/sdks/python/apache_beam/io/tfrecordio_test.py
index b7e370d0187a..3c70ade3dbe3 100644
--- a/sdks/python/apache_beam/io/tfrecordio_test.py
+++ b/sdks/python/apache_beam/io/tfrecordio_test.py
@@ -36,6 +36,8 @@
 from apache_beam.io.tfrecordio import ReadFromTFRecord
 from apache_beam.io.tfrecordio import WriteToTFRecord
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 import crcmod
 
 
@@ -254,7 +256,7 @@ def test_process_single(self):
                         coder=coders.BytesCoder(),
                         compression_type=CompressionTypes.AUTO,
                         validate=True)))
-      beam.assert_that(result, beam.equal_to(['foo']))
+      assert_that(result, equal_to(['foo']))
 
   def test_process_multiple(self):
     path = os.path.join(self._new_tempdir(), 'result')
@@ -267,7 +269,7 @@ def test_process_multiple(self):
                         coder=coders.BytesCoder(),
                         compression_type=CompressionTypes.AUTO,
                         validate=True)))
-      beam.assert_that(result, beam.equal_to(['foo', 'bar']))
+      assert_that(result, equal_to(['foo', 'bar']))
 
   def test_process_gzip(self):
     path = os.path.join(self._new_tempdir(), 'result')
@@ -280,7 +282,7 @@ def test_process_gzip(self):
                         coder=coders.BytesCoder(),
                         compression_type=CompressionTypes.GZIP,
                         validate=True)))
-      beam.assert_that(result, beam.equal_to(['foo', 'bar']))
+      assert_that(result, equal_to(['foo', 'bar']))
 
   def test_process_auto(self):
     path = os.path.join(self._new_tempdir(), 'result.gz')
@@ -293,7 +295,7 @@ def test_process_auto(self):
                         coder=coders.BytesCoder(),
                         compression_type=CompressionTypes.AUTO,
                         validate=True)))
-      beam.assert_that(result, beam.equal_to(['foo', 'bar']))
+      assert_that(result, equal_to(['foo', 'bar']))
 
 
 class TestReadFromTFRecordSource(TestTFRecordSource):
@@ -305,7 +307,7 @@ def test_process_gzip(self):
       result = (p
                 | ReadFromTFRecord(
                     path, compression_type=CompressionTypes.GZIP))
-      beam.assert_that(result, beam.equal_to(['foo', 'bar']))
+      assert_that(result, equal_to(['foo', 'bar']))
 
   def test_process_gzip_auto(self):
     path = os.path.join(self._new_tempdir(), 'result.gz')
@@ -314,7 +316,7 @@ def test_process_gzip_auto(self):
       result = (p
                 | ReadFromTFRecord(
                     path, compression_type=CompressionTypes.AUTO))
-      beam.assert_that(result, beam.equal_to(['foo', 'bar']))
+      assert_that(result, equal_to(['foo', 'bar']))
 
 
 class TestEnd2EndWriteAndRead(_TestCaseWithTempDirCleanUp):
@@ -337,7 +339,7 @@ def test_end2end(self):
     # Read the file back and compare.
     with TestPipeline() as p:
       actual_data = p | ReadFromTFRecord(file_path_prefix + '-*')
-      beam.assert_that(actual_data, beam.equal_to(expected_data))
+      assert_that(actual_data, equal_to(expected_data))
 
   def test_end2end_auto_compression(self):
     file_path_prefix = os.path.join(self._new_tempdir(), 'result')
@@ -351,7 +353,7 @@ def test_end2end_auto_compression(self):
     # Read the file back and compare.
     with TestPipeline() as p:
       actual_data = p | ReadFromTFRecord(file_path_prefix + '-*')
-      beam.assert_that(actual_data, beam.equal_to(expected_data))
+      assert_that(actual_data, equal_to(expected_data))
 
   def test_end2end_auto_compression_unsharded(self):
     file_path_prefix = os.path.join(self._new_tempdir(), 'result')
@@ -365,7 +367,7 @@ def test_end2end_auto_compression_unsharded(self):
     # Read the file back and compare.
     with TestPipeline() as p:
       actual_data = p | ReadFromTFRecord(file_path_prefix + '.gz')
-      beam.assert_that(actual_data, beam.equal_to(expected_data))
+      assert_that(actual_data, equal_to(expected_data))
 
   @unittest.skipIf(tf is None, 'tensorflow not installed.')
   def test_end2end_example_proto(self):
@@ -385,7 +387,7 @@ def test_end2end_example_proto(self):
       actual_data = (p | ReadFromTFRecord(
           file_path_prefix + '-*',
           coder=beam.coders.ProtoCoder(example.__class__)))
-      beam.assert_that(actual_data, beam.equal_to([example]))
+      assert_that(actual_data, equal_to([example]))
 
   def test_end2end_read_write_read(self):
     path = os.path.join(self._new_tempdir(), 'result')
@@ -400,7 +402,7 @@ def test_end2end_read_write_read(self):
     # Read the file back and compare.
     with TestPipeline() as p:
       actual_data = p | ReadFromTFRecord(path+'-*', validate=True)
-      beam.assert_that(actual_data, beam.equal_to(expected_data))
+      assert_that(actual_data, equal_to(expected_data))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py
index 8aa8a8aa0292..e0775d103681 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -33,6 +33,8 @@
 from apache_beam.pvalue import AsSingleton
 from apache_beam.runners.dataflow.native_io.iobase import NativeSource
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 from apache_beam.transforms import CombineGlobally
 from apache_beam.transforms import Create
 from apache_beam.transforms import FlatMap
@@ -41,8 +43,6 @@
 from apache_beam.transforms import ParDo
 from apache_beam.transforms import PTransform
 from apache_beam.transforms import WindowInto
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
 from apache_beam.transforms.window import SlidingWindows
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.utils.timestamp import MIN_TIMESTAMP
diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
index 062e6f9bef16..b7ba15a502aa 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py
@@ -28,9 +28,9 @@
 from apache_beam.metrics.metricbase import MetricName
 
 from apache_beam.pvalue import AsList
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import BeamAssertException
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import BeamAssertException
+from apache_beam.testing.util import equal_to
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.runners.portability import maptask_executor_runner
 
diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py
index c61c49f77916..fa80b1c9555c 100644
--- a/sdks/python/apache_beam/runners/runner_test.py
+++ b/sdks/python/apache_beam/runners/runner_test.py
@@ -36,8 +36,8 @@
 from apache_beam.pipeline import Pipeline
 from apache_beam.runners import DirectRunner
 from apache_beam.runners import create_runner
-from apache_beam.transforms.util import assert_that
-from apache_beam.transforms.util import equal_to
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
 from apache_beam.options.pipeline_options import PipelineOptions
 
 
diff --git a/sdks/python/apache_beam/testing/util.py b/sdks/python/apache_beam/testing/util.py
new file mode 100644
index 000000000000..60a6b21ca39b
--- /dev/null
+++ b/sdks/python/apache_beam/testing/util.py
@@ -0,0 +1,107 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Utilities for testing Beam pipelines."""
+
+from __future__ import absolute_import
+
+from apache_beam import pvalue
+from apache_beam.transforms import window
+from apache_beam.transforms.core import Create
+from apache_beam.transforms.core import Map
+from apache_beam.transforms.core import WindowInto
+from apache_beam.transforms.util import CoGroupByKey
+from apache_beam.transforms.ptransform import PTransform
+
+
+__all__ = [
+    'assert_that',
+    'equal_to',
+    'is_empty',
+    ]
+
+
+class BeamAssertException(Exception):
+  """Exception raised by matcher classes used by assert_that transform."""
+
+  pass
+
+
+# Note that equal_to always sorts the expected and actual since what we
+# compare are PCollections for which there is no guaranteed order.
+# However the sorting does not go beyond top level therefore [1,2] and [2,1]
+# are considered equal and [[1,2]] and [[2,1]] are not.
+# TODO(silviuc): Add contains_in_any_order-style matchers.
+def equal_to(expected):
+  expected = list(expected)
+
+  def _equal(actual):
+    sorted_expected = sorted(expected)
+    sorted_actual = sorted(actual)
+    if sorted_expected != sorted_actual:
+      raise BeamAssertException(
+          'Failed assert: %r == %r' % (sorted_expected, sorted_actual))
+  return _equal
+
+
+def is_empty():
+  def _empty(actual):
+    actual = list(actual)
+    if actual:
+      raise BeamAssertException(
+          'Failed assert: [] == %r' % actual)
+  return _empty
+
+
+def assert_that(actual, matcher, label='assert_that'):
+  """A PTransform that checks a PCollection has an expected value.
+
+  Note that assert_that should be used only for testing pipelines since the
+  check relies on materializing the entire PCollection being checked.
+
+  Args:
+    actual: A PCollection.
+    matcher: A matcher function taking as argument the actual value of a
+      materialized PCollection. The matcher validates this actual value against
+      expectations and raises BeamAssertException if they are not met.
+    label: Optional string label. This is needed in case several assert_that
+      transforms are introduced in the same pipeline.
+
+  Returns:
+    Ignored.
+  """
+  assert isinstance(actual, pvalue.PCollection)
+
+  class AssertThat(PTransform):
+
+    def expand(self, pcoll):
+      # We must have at least a single element to ensure the matcher
+      # code gets run even if the input pcollection is empty.
+      keyed_singleton = pcoll.pipeline | Create([(None, None)])
+      keyed_actual = (
+          pcoll
+          | WindowInto(window.GlobalWindows())
+          | "ToVoidKey" >> Map(lambda v: (None, v)))
+      _ = ((keyed_singleton, keyed_actual)
+           | "Group" >> CoGroupByKey()
+           | "Unkey" >> Map(lambda (k, (_, actual_values)): actual_values)
+           | "Match" >> Map(matcher))
+
+    def default_label(self):
+      return label
+
+  actual | AssertThat()  # pylint: disable=expression-not-assigned
diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/testing/util_test.py
similarity index 93%
rename from sdks/python/apache_beam/transforms/util_test.py
rename to sdks/python/apache_beam/testing/util_test.py
index 7fdef702871f..1acebb651c55 100644
--- a/sdks/python/apache_beam/transforms/util_test.py
+++ b/sdks/python/apache_beam/testing/util_test.py
@@ -15,13 +15,13 @@
 # limitations under the License.
 #
 
-"""Unit tests for the util transforms."""
+"""Unit tests for testing utilities."""
 
 import unittest
 
 from apache_beam import Create
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that, equal_to, is_empty
+from apache_beam.testing.util import assert_that, equal_to, is_empty
 
 
 class UtilTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
index 1822c1911927..946a60a3675c 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -24,13 +24,13 @@
 import apache_beam as beam
 from apache_beam.testing.test_pipeline import TestPipeline
 import apache_beam.transforms.combiners as combine
+from apache_beam.testing.util import assert_that, equal_to
 from apache_beam.transforms.core import CombineGlobally
 from apache_beam.transforms.core import Create
 from apache_beam.transforms.core import Map
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
 from apache_beam.transforms.ptransform import PTransform
-from apache_beam.transforms.util import assert_that, equal_to
 
 
 class CombineTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/create_test.py b/sdks/python/apache_beam/transforms/create_test.py
index 9ede4c7d2d41..55ad7f32982d 100644
--- a/sdks/python/apache_beam/transforms/create_test.py
+++ b/sdks/python/apache_beam/transforms/create_test.py
@@ -20,9 +20,10 @@
 
 from apache_beam.io import source_test_utils
 
-from apache_beam import Create, assert_that, equal_to
+from apache_beam import Create
 from apache_beam.coders import FastPrimitivesCoder
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that, equal_to
 
 
 class CreateTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index 3320d79bbbb0..f7906606c69c 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -33,12 +33,12 @@
 from apache_beam.options.pipeline_options import TypeOptions
 import apache_beam.pvalue as pvalue
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that, equal_to
 from apache_beam.transforms import window
 from apache_beam.transforms.core import GroupByKeyOnly
 import apache_beam.transforms.combiners as combine
 from apache_beam.transforms.display import DisplayData, DisplayDataItem
 from apache_beam.transforms.ptransform import PTransform
-from apache_beam.transforms.util import assert_that, equal_to
 import apache_beam.typehints as typehints
 from apache_beam.typehints import with_input_types
 from apache_beam.typehints import with_output_types
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py
index 0bc910799571..650068141852 100644
--- a/sdks/python/apache_beam/transforms/sideinputs_test.py
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -24,8 +24,8 @@
 
 import apache_beam as beam
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that, equal_to
 from apache_beam.transforms import window
-from apache_beam.transforms.util import assert_that, equal_to
 
 
 class SideInputsTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index da7f43bdf82a..4a5affb6a30a 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -27,6 +27,7 @@
 import apache_beam as beam
 from apache_beam.runners import pipeline_context
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that, equal_to
 from apache_beam.transforms import trigger
 from apache_beam.transforms.core import Windowing
 from apache_beam.transforms.trigger import AccumulationMode
@@ -40,7 +41,6 @@
 from apache_beam.transforms.trigger import InMemoryUnmergedState
 from apache_beam.transforms.trigger import Repeatedly
 from apache_beam.transforms.trigger import TriggerFn
-from apache_beam.transforms.util import assert_that, equal_to
 from apache_beam.transforms.window import FixedWindows
 from apache_beam.transforms.window import IntervalWindow
 from apache_beam.transforms.window import MIN_TIMESTAMP
diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py
index a6ecf0a700d1..a7484acb2cc7 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -20,14 +20,10 @@
 
 from __future__ import absolute_import
 
-from apache_beam import pvalue
-from apache_beam.transforms import window
 from apache_beam.transforms.core import CombinePerKey
-from apache_beam.transforms.core import Create
 from apache_beam.transforms.core import Flatten
 from apache_beam.transforms.core import GroupByKey
 from apache_beam.transforms.core import Map
-from apache_beam.transforms.core import WindowInto
 from apache_beam.transforms.ptransform import PTransform
 from apache_beam.transforms.ptransform import ptransform_fn
 
@@ -38,9 +34,6 @@
     'KvSwap',
     'RemoveDuplicates',
     'Values',
-    'assert_that',
-    'equal_to',
-    'is_empty',
     ]
 
 
@@ -169,75 +162,3 @@ def RemoveDuplicates(pcoll):  # pylint: disable=invalid-name
           | 'ToPairs' >> Map(lambda v: (v, None))
           | 'Group' >> CombinePerKey(lambda vs: None)
           | 'RemoveDuplicates' >> Keys())
-
-
-class BeamAssertException(Exception):
-  """Exception raised by matcher classes used by assert_that transform."""
-
-  pass
-
-
-# Note that equal_to always sorts the expected and actual since what we
-# compare are PCollections for which there is no guaranteed order.
-# However the sorting does not go beyond top level therefore [1,2] and [2,1]
-# are considered equal and [[1,2]] and [[2,1]] are not.
-# TODO(silviuc): Add contains_in_any_order-style matchers.
-def equal_to(expected):
-  expected = list(expected)
-
-  def _equal(actual):
-    sorted_expected = sorted(expected)
-    sorted_actual = sorted(actual)
-    if sorted_expected != sorted_actual:
-      raise BeamAssertException(
-          'Failed assert: %r == %r' % (sorted_expected, sorted_actual))
-  return _equal
-
-
-def is_empty():
-  def _empty(actual):
-    actual = list(actual)
-    if actual:
-      raise BeamAssertException(
-          'Failed assert: [] == %r' % actual)
-  return _empty
-
-
-def assert_that(actual, matcher, label='assert_that'):
-  """A PTransform that checks a PCollection has an expected value.
-
-  Note that assert_that should be used only for testing pipelines since the
-  check relies on materializing the entire PCollection being checked.
-
-  Args:
-    actual: A PCollection.
-    matcher: A matcher function taking as argument the actual value of a
-      materialized PCollection. The matcher validates this actual value against
-      expectations and raises BeamAssertException if they are not met.
-    label: Optional string label. This is needed in case several assert_that
-      transforms are introduced in the same pipeline.
-
-  Returns:
-    Ignored.
-  """
-  assert isinstance(actual, pvalue.PCollection)
-
-  class AssertThat(PTransform):
-
-    def expand(self, pcoll):
-      # We must have at least a single element to ensure the matcher
-      # code gets run even if the input pcollection is empty.
-      keyed_singleton = pcoll.pipeline | Create([(None, None)])
-      keyed_actual = (
-          pcoll
-          | WindowInto(window.GlobalWindows())
-          | "ToVoidKey" >> Map(lambda v: (None, v)))
-      _ = ((keyed_singleton, keyed_actual)
-           | "Group" >> CoGroupByKey()
-           | "Unkey" >> Map(lambda (k, (_, actual_values)): actual_values)
-           | "Match" >> Map(matcher))
-
-    def default_label(self):
-      return label
-
-  actual | AssertThat()  # pylint: disable=expression-not-assigned
diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py
index a7797dd435d9..fd1bb9d5250b 100644
--- a/sdks/python/apache_beam/transforms/window_test.py
+++ b/sdks/python/apache_beam/transforms/window_test.py
@@ -21,6 +21,7 @@
 
 from apache_beam.runners import pipeline_context
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that, equal_to
 from apache_beam.transforms import CombinePerKey
 from apache_beam.transforms import combiners
 from apache_beam.transforms import core
@@ -31,7 +32,6 @@
 from apache_beam.transforms.core import Windowing
 from apache_beam.transforms.trigger import AccumulationMode
 from apache_beam.transforms.trigger import AfterCount
-from apache_beam.transforms.util import assert_that, equal_to
 from apache_beam.transforms.window import FixedWindows
 from apache_beam.transforms.window import GlobalWindow
 from apache_beam.transforms.window import GlobalWindows
diff --git a/sdks/python/apache_beam/transforms/write_ptransform_test.py b/sdks/python/apache_beam/transforms/write_ptransform_test.py
index 27e7caa8e5a7..e31b9cc9830d 100644
--- a/sdks/python/apache_beam/transforms/write_ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/write_ptransform_test.py
@@ -23,8 +23,8 @@
 
 from apache_beam.io import iobase
 from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that, is_empty
 from apache_beam.transforms.ptransform import PTransform
-from apache_beam.transforms.util import assert_that, is_empty
 
 
 class _TestSink(iobase.Sink):
diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
index 3494cfe1e6a8..589dc0e5ac82 100644
--- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
+++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
@@ -25,7 +25,7 @@
 from apache_beam import typehints
 from apache_beam.options.pipeline_options import OptionsContext
 from apache_beam.testing.test_pipeline import TestPipeline
-from apache_beam.transforms.util import assert_that, equal_to
+from apache_beam.testing.util import assert_that, equal_to
 from apache_beam.typehints import WithTypeHints
 
 # These test often construct a pipeline as value | PTransform to test side

From 98e685d85a4cf75faee8156516d7d3aff60077f3 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@google.com>
Date: Wed, 10 May 2017 15:55:46 -0700
Subject: [PATCH 215/387] Remove some internal details from the public API.

---
 sdks/python/apache_beam/pipeline.py           |  4 ++--
 .../runners/dataflow/dataflow_runner.py       |  6 ++---
 .../runners/dataflow/dataflow_runner_test.py  |  6 ++---
 .../apache_beam/runners/direct/executor.py    |  2 +-
 .../runners/direct/transform_evaluator.py     | 10 ++++-----
 .../python/apache_beam/transforms/__init__.py |  2 +-
 sdks/python/apache_beam/transforms/core.py    | 22 +++++++++----------
 .../apache_beam/transforms/ptransform.py      |  6 ++---
 .../apache_beam/transforms/ptransform_test.py | 12 +++++-----
 .../python/apache_beam/typehints/typecheck.py |  2 +-
 10 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index 79480d77f194..5048534b3852 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -77,8 +77,8 @@ class Pipeline(object):
   the PValues are the edges.
 
   All the transforms applied to the pipeline must have distinct full labels.
-  If same transform instance needs to be applied then a clone should be created
-  with a new label (e.g., transform.clone('new label')).
+  If same transform instance needs to be applied then the right shift operator
+  should be used to designate new names (e.g. `input | "label" >> my_tranform`).
   """
 
   def __init__(self, runner=None, options=None, argv=None):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index da8de9d1dce0..0ecd22ac80b2 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -160,7 +160,7 @@ def group_by_key_input_visitor():
 
     class GroupByKeyInputVisitor(PipelineVisitor):
       """A visitor that replaces `Any` element type for input `PCollection` of
-      a `GroupByKey` or `GroupByKeyOnly` with a `KV` type.
+      a `GroupByKey` or `_GroupByKeyOnly` with a `KV` type.
 
       TODO(BEAM-115): Once Python SDk is compatible with the new Runner API,
       we could directly replace the coder instead of mutating the element type.
@@ -169,8 +169,8 @@ class GroupByKeyInputVisitor(PipelineVisitor):
       def visit_transform(self, transform_node):
         # Imported here to avoid circular dependencies.
         # pylint: disable=wrong-import-order, wrong-import-position
-        from apache_beam.transforms.core import GroupByKey, GroupByKeyOnly
-        if isinstance(transform_node.transform, (GroupByKey, GroupByKeyOnly)):
+        from apache_beam.transforms.core import GroupByKey, _GroupByKeyOnly
+        if isinstance(transform_node.transform, (GroupByKey, _GroupByKeyOnly)):
           pcoll = transform_node.inputs[0]
           input_type = pcoll.element_type
           # If input_type is not specified, then treat it as `Any`.
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
index ac9b0282fbcc..ff4b51d5a826 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py
@@ -37,7 +37,7 @@
 from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayDataItem
-from apache_beam.transforms.core import GroupByKeyOnly
+from apache_beam.transforms.core import _GroupByKeyOnly
 from apache_beam.typehints import typehints
 
 # Protect against environments where apitools library is not available.
@@ -185,7 +185,7 @@ def test_group_by_key_input_visitor_with_valid_inputs(self):
     pcoll1 = PCollection(p)
     pcoll2 = PCollection(p)
     pcoll3 = PCollection(p)
-    for transform in [GroupByKeyOnly(), beam.GroupByKey()]:
+    for transform in [_GroupByKeyOnly(), beam.GroupByKey()]:
       pcoll1.element_type = None
       pcoll2.element_type = typehints.Any
       pcoll3.element_type = typehints.KV[typehints.Any, typehints.Any]
@@ -199,7 +199,7 @@ def test_group_by_key_input_visitor_with_invalid_inputs(self):
     p = TestPipeline()
     pcoll1 = PCollection(p)
     pcoll2 = PCollection(p)
-    for transform in [GroupByKeyOnly(), beam.GroupByKey()]:
+    for transform in [_GroupByKeyOnly(), beam.GroupByKey()]:
       pcoll1.element_type = typehints.TupleSequenceConstraint
       pcoll2.element_type = typehints.Set
       err_msg = "Input to GroupByKey must be of Tuple or Any type"
diff --git a/sdks/python/apache_beam/runners/direct/executor.py b/sdks/python/apache_beam/runners/direct/executor.py
index 9efbede47e9b..86db29159f37 100644
--- a/sdks/python/apache_beam/runners/direct/executor.py
+++ b/sdks/python/apache_beam/runners/direct/executor.py
@@ -155,7 +155,7 @@ class _SerialEvaluationState(_TransformEvaluationState):
   item of work will be submitted to the ExecutorService at any time.
 
   A principal use of this is for evaluators that keeps a global state such as
-  GroupByKeyOnly.
+  _GroupByKeyOnly.
   """
 
   def __init__(self, executor_service, scheduled):
diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
index 6984ded7cc3f..b1cb626ca0cb 100644
--- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py
+++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
@@ -53,7 +53,7 @@ def __init__(self, evaluation_context):
         io.Read: _BoundedReadEvaluator,
         core.Flatten: _FlattenEvaluator,
         core.ParDo: _ParDoEvaluator,
-        core.GroupByKeyOnly: _GroupByKeyOnlyEvaluator,
+        core._GroupByKeyOnly: _GroupByKeyOnlyEvaluator,
         _NativeWrite: _NativeWriteEvaluator,
     }
 
@@ -83,7 +83,7 @@ def should_execute_serially(self, applied_ptransform):
     """Returns True if this applied_ptransform should run one bundle at a time.
 
     Some TransformEvaluators use a global state object to keep track of their
-    global execution state. For example evaluator for GroupByKeyOnly uses this
+    global execution state. For example evaluator for _GroupByKeyOnly uses this
     state as an in memory dictionary to buffer keys.
 
     Serially executed evaluators will act as syncing point in the graph and
@@ -99,7 +99,7 @@ def should_execute_serially(self, applied_ptransform):
       True if executor should execute applied_ptransform serially.
     """
     return isinstance(applied_ptransform.transform,
-                      (core.GroupByKeyOnly, _NativeWrite))
+                      (core._GroupByKeyOnly, _NativeWrite))
 
 
 class _TransformEvaluator(object):
@@ -325,7 +325,7 @@ def finish_bundle(self):
 
 
 class _GroupByKeyOnlyEvaluator(_TransformEvaluator):
-  """TransformEvaluator for GroupByKeyOnly transform."""
+  """TransformEvaluator for _GroupByKeyOnly transform."""
 
   MAX_ELEMENT_PER_BUNDLE = None
 
@@ -369,7 +369,7 @@ def process_element(self, element):
       k, v = element.value
       self.state.output[self.key_coder.encode(k)].append(v)
     else:
-      raise TypeCheckError('Input to GroupByKeyOnly must be a PCollection of '
+      raise TypeCheckError('Input to _GroupByKeyOnly must be a PCollection of '
                            'windowed key-value pairs. Instead received: %r.'
                            % element)
 
diff --git a/sdks/python/apache_beam/transforms/__init__.py b/sdks/python/apache_beam/transforms/__init__.py
index 847fb8ffb9cb..b77b0f65646d 100644
--- a/sdks/python/apache_beam/transforms/__init__.py
+++ b/sdks/python/apache_beam/transforms/__init__.py
@@ -21,5 +21,5 @@
 from apache_beam.transforms import combiners
 from apache_beam.transforms.core import *
 from apache_beam.transforms.ptransform import *
-from apache_beam.transforms.timeutil import *
+from apache_beam.transforms.timeutil import TimeDomain
 from apache_beam.transforms.util import *
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index abe699ff1607..0e497f9448d1 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -215,7 +215,7 @@ def _strip_output_annotations(self, type_hint):
       return Any
     return type_hint
 
-  def process_argspec_fn(self):
+  def _process_argspec_fn(self):
     """Returns the Python callable that will eventually be invoked.
 
     This should ideally be the user-level function that is called with
@@ -307,7 +307,7 @@ def infer_output_type(self, input_type):
     return self._strip_output_annotations(
         trivial_inference.infer_return_type(self._fn, [input_type]))
 
-  def process_argspec_fn(self):
+  def _process_argspec_fn(self):
     return getattr(self._fn, '_argspec_fn', self._fn)
 
 
@@ -641,8 +641,8 @@ def make_fn(self, fn):
       return fn
     return CallableWrapperDoFn(fn)
 
-  def process_argspec_fn(self):
-    return self.fn.process_argspec_fn()
+  def _process_argspec_fn(self):
+    return self.fn._process_argspec_fn()
 
   def display_data(self):
     return {'fn': DisplayDataItem(self.fn.__class__,
@@ -870,19 +870,19 @@ def display_data(self):
   def default_label(self):
     return 'CombineGlobally(%s)' % ptransform.label_from_callable(self.fn)
 
-  def clone(self, **extra_attributes):
+  def _clone(self, **extra_attributes):
     clone = copy.copy(self)
     clone.__dict__.update(extra_attributes)
     return clone
 
   def with_defaults(self, has_defaults=True):
-    return self.clone(has_defaults=has_defaults)
+    return self._clone(has_defaults=has_defaults)
 
   def without_defaults(self):
     return self.with_defaults(False)
 
   def as_singleton_view(self):
-    return self.clone(as_view=True)
+    return self._clone(as_view=True)
 
   def expand(self, pcoll):
     def add_input_types(transform):
@@ -964,7 +964,7 @@ def make_fn(self, fn):
   def default_label(self):
     return '%s(%s)' % (self.__class__.__name__, self._fn_label)
 
-  def process_argspec_fn(self):
+  def _process_argspec_fn(self):
     return self.fn._fn  # pylint: disable=protected-access
 
   def expand(self, pcoll):
@@ -1133,7 +1133,7 @@ def expand(self, pcoll):
       return (pcoll
               | 'ReifyWindows' >> (ParDo(self.ReifyWindows())
                  .with_output_types(reify_output_type))
-              | 'GroupByKey' >> (GroupByKeyOnly()
+              | 'GroupByKey' >> (_GroupByKeyOnly()
                  .with_input_types(reify_output_type)
                  .with_output_types(gbk_input_type))
               | ('GroupByWindow' >> ParDo(
@@ -1144,14 +1144,14 @@ def expand(self, pcoll):
       # The input_type is None, run the default
       return (pcoll
               | 'ReifyWindows' >> ParDo(self.ReifyWindows())
-              | 'GroupByKey' >> GroupByKeyOnly()
+              | 'GroupByKey' >> _GroupByKeyOnly()
               | 'GroupByWindow' >> ParDo(
                     self.GroupAlsoByWindow(pcoll.windowing)))
 
 
 @typehints.with_input_types(typehints.KV[K, V])
 @typehints.with_output_types(typehints.KV[K, typehints.Iterable[V]])
-class GroupByKeyOnly(PTransform):
+class _GroupByKeyOnly(PTransform):
   """A group by key transform, ignoring windows."""
   def infer_output_type(self, input_type):
     key_type, value_type = trivial_inference.key_value_types(input_type)
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index 8898c36253fc..bd2a120b8c63 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -303,7 +303,7 @@ def _infer_output_coder(self, input_type=None, input_coder=None):
     # TODO(ccy): further refine this API.
     return None
 
-  def clone(self, new_label):
+  def _clone(self, new_label):
     """Clones the current transform instance under a new label."""
     transform = copy.copy(self)
     transform.label = new_label
@@ -567,7 +567,7 @@ def element_type(side_input):
 
       arg_types = [pvalueish.element_type] + [element_type(v) for v in args]
       kwargs_types = {k: element_type(v) for (k, v) in kwargs.items()}
-      argspec_fn = self.process_argspec_fn()
+      argspec_fn = self._process_argspec_fn()
       bindings = getcallargs_forhints(argspec_fn, *arg_types, **kwargs_types)
       hints = getcallargs_forhints(argspec_fn, *type_hints[0], **type_hints[1])
       for arg, hint in hints.items():
@@ -581,7 +581,7 @@ def element_type(side_input):
               'Type hint violation for \'%s\': requires %s but got %s for %s'
               % (self.label, hint, bindings[arg], arg))
 
-  def process_argspec_fn(self):
+  def _process_argspec_fn(self):
     """Returns an argspec of the function actually consuming the data.
     """
     raise NotImplementedError
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index f7906606c69c..efc597877b29 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -35,7 +35,7 @@
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.testing.util import assert_that, equal_to
 from apache_beam.transforms import window
-from apache_beam.transforms.core import GroupByKeyOnly
+from apache_beam.transforms.core import _GroupByKeyOnly
 import apache_beam.transforms.combiners as combine
 from apache_beam.transforms.display import DisplayData, DisplayDataItem
 from apache_beam.transforms.ptransform import PTransform
@@ -580,7 +580,7 @@ def test_group_by_key_only_input_must_be_kv_pairs(self):
     pipeline = TestPipeline()
     pcolls = pipeline | 'A' >> beam.Create(['a', 'b', 'f'])
     with self.assertRaises(typehints.TypeCheckError) as cm:
-      pcolls | 'D' >> GroupByKeyOnly()
+      pcolls | 'D' >> _GroupByKeyOnly()
       pipeline.run()
 
     expected_error_prefix = ('Input type hint violation at D: expected '
@@ -1088,7 +1088,7 @@ def test_group_by_key_only_output_type_deduction(self):
          | 'Str' >> beam.Create(['t', 'e', 's', 't']).with_output_types(str)
          | ('Pair' >> beam.Map(lambda x: (x, ord(x)))
             .with_output_types(typehints.KV[str, str]))
-         | GroupByKeyOnly())
+         | _GroupByKeyOnly())
 
     # Output type should correctly be deduced.
     # GBK-only should deduce that KV[A, B] is turned into KV[A, Iterable[B]].
@@ -1112,7 +1112,7 @@ def test_group_by_key_only_does_not_type_check(self):
     with self.assertRaises(typehints.TypeCheckError) as e:
       (self.p
        | beam.Create([1, 2, 3]).with_output_types(int)
-       | 'F' >> GroupByKeyOnly())
+       | 'F' >> _GroupByKeyOnly())
 
     self.assertEqual("Input type hint violation at F: "
                      "expected Tuple[TypeVariable[K], TypeVariable[V]], "
@@ -1155,7 +1155,7 @@ def test_pipeline_checking_gbk_insufficient_type_information(self):
       (self.p
        | 'Nums' >> beam.Create(range(5)).with_output_types(int)
        | 'ModDup' >> beam.Map(lambda x: (x % 2, x))
-       | GroupByKeyOnly())
+       | _GroupByKeyOnly())
 
     self.assertEqual('Pipeline type checking is enabled, however no output '
                      'type-hint was found for the PTransform '
@@ -1978,7 +1978,7 @@ def test_pardo_type_inference(self):
   def test_gbk_type_inference(self):
     self.assertEqual(
         typehints.Tuple[str, typehints.Iterable[int]],
-        GroupByKeyOnly().infer_output_type(typehints.KV[str, int]))
+        _GroupByKeyOnly().infer_output_type(typehints.KV[str, int]))
 
   def test_pipeline_inference(self):
     created = self.p | beam.Create(['a', 'b', 'c'])
diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py
index 09b73f994ddd..89a5f5c7e2cf 100644
--- a/sdks/python/apache_beam/typehints/typecheck.py
+++ b/sdks/python/apache_beam/typehints/typecheck.py
@@ -109,7 +109,7 @@ class TypeCheckWrapperDoFn(AbstractDoFnWrapper):
   def __init__(self, dofn, type_hints, label=None):
     super(TypeCheckWrapperDoFn, self).__init__(dofn)
     self.dofn = dofn
-    self._process_fn = self.dofn.process_argspec_fn()
+    self._process_fn = self.dofn._process_argspec_fn()
     if type_hints.input_types:
       input_args, input_kwargs = type_hints.input_types
       self._input_hints = getcallargs_forhints(

From d76349beeab5cf6e0513d6272d635ac0b4662239 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 11 May 2017 14:03:08 -0700
Subject: [PATCH 216/387] Don't deploy jdk1.8-tests module

---
 pom.xml                                  | 6 ++++++
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/pom.xml b/pom.xml
index e7eb729b6d35..eaca6b7ff661 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1177,6 +1177,12 @@
           </executions>
         </plugin>
 
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-deploy-plugin</artifactId>
+          <version>2.8.2</version>
+        </plugin>
+
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-jar-plugin</artifactId>
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
index 56a7454a3746..9f84e881de0c 100644
--- a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
+++ b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml
@@ -80,6 +80,13 @@
           <parallel>none</parallel>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-deploy-plugin</artifactId>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
   <!--The dataflow-runner and spark-runner profiles support using those runners

From fc3e28de37acc9de597518cf9fe793d1ddebcf2a Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 11 May 2017 15:47:54 -0700
Subject: [PATCH 217/387] Fix due to GBKO name change.

---
 .../apache_beam/runners/portability/maptask_executor_runner.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
index 077871ed50ca..ddfc4ccc43f1 100644
--- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
+++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py
@@ -243,7 +243,7 @@ def run_side_write(self, pcoll, label):
         (label, write_sideinput_op))
     return output_buffer
 
-  def run_GroupByKeyOnly(self, transform_node):
+  def run__GroupByKeyOnly(self, transform_node):
     map_task_index, producer_index, output_index = self.outputs[
         transform_node.inputs[0]]
     grouped_element_coder = self._get_coder(transform_node.outputs[None],

From a45d63f6362e88d034704a4749cef7eedc76a691 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 11 May 2017 16:41:06 -0700
Subject: [PATCH 218/387] Remove unused test data

---
 .../apache_beam/testing/data/privatekey.p12      | Bin 2452 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 sdks/python/apache_beam/testing/data/privatekey.p12

diff --git a/sdks/python/apache_beam/testing/data/privatekey.p12 b/sdks/python/apache_beam/testing/data/privatekey.p12
deleted file mode 100644
index c369ecb6e664cafddf5a817ce0d50c2059867042..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2452
zcmY+EXHXLg7KH-|p#~6;Zh#;?LLl^N04dT%0VyIVB@~sKfJg~d>XQ=C_$Xp1(jHBb
zVg%`iM<~(~2u+j@60Cr$GrRkCf84n<-<di8zJtKA|G~@zMBvz+!0d`C7AapKOsq_$
zICgUoj@{%JHbUTl^nW67DGo^ag~-fIfM2igpM(h^4(9lG0~Zq*!3|=QBj0r4yw|GA
z%*+Oe#{rAXs7l7RArGODUhnb7II{|8PYB#<zvR@yf%*x~OJ_N9Q!IR6+?;q+p<{bE
zcEVq%LJ@>2Qm)a_<XY(@kz>MO%$5&P0=1A39Se>1SNp@uq-7L4_Q-l?bf_v-R23-{
z#0g84zkr=m0Ajj4&*H|+g(3*yYHWh!s2f;6d_zWD9Ch-~QB_&pvbP*gE4(L2-zuS4
z;@FnvYy<zqwd{JdDJDNywmu2-XTZ7G(Imp*<W^?xGJXAR%uMv>`wF7`jst@`c1VtQ
z82S9v(QiV2$u4_Nl@4>QDnDCahX$=_Kfz#^DhlKd`@jqc<yP4%wcK<-#YW3Fu}qMV
zFDQ<>b@fwa%M(NJPu-(O^ck<un0D-xHHS0<$GTd0mYOhD5sBZ>0Hd_Kz@AvKLB7^$
z6JADn8Z&SA@qB%6WM~7=?OMNbLkrqhV?Ed)(hl-=F$e2G;PZ6>ivMWCE1VCsvCNfE
zjezr6xIGH!l58QCb%^$lzSDfEL(%#Em9lE-KEh*()UmnepG(VFy%)C$Z}qp0#`#Ni
z3DQ2PEIzU)?bz`<6yN(|K32V0<hFWU7hMoG`(5HMy0J7gD37oosYn)lFEJorR+yr?
zhjr11uR5@<>qRWbrJ0dVPSmpFl{K=8=8IK&87PO3N|&!qBu2FI$5BF#aip}Q`|#RR
z2Hsl0WO1)O_BYG2@tb~^GL!OE4tgG#!5%Iy)ekfA?Rc!F!Or2Oi}SN4!yuy;fwz^G
zskz1quE|+Lvv^+-!8J=-BR=Qog$@nGUH2uk9A*F;h|Gma_unr_nhv3lah1MD*9FV}
zfX>XGcSiFyccshm3VuZSOY7)v#oY9q)XX_E-Bw&p8mbDoZJIY%{4;(+N^aM}A}r2s
zsW7qYLH^r?>7kUKW&3q(Pdhu=M^<)Cm?-d9Pvv={s3=OjDQmlOpC)TwDx6${GPXo8
zwTBi(eI1N2T#jU<sj(EMbPRD&n>y_;tI)OwIPwBRwX}=?sH2oLym;n2p!+jq9j~Wv
zMBW(z?#?S^Mw5@XFj$jwVj^4-7IqhV4Tm#sRf5&wAAAiR;=c#B#wZyQ$1Ud@H{115
z+j?@(OAKi)AkI^!S_ZT(D+Thh%SnI51Ir9%-r6rq)(G*&moM&Rmn*ki9G}y;E7=%%
z9}<|w&B-^BR{1Jpa@xeemBWmju51jM`r=S}``Qhb1=Xi@!;2*d9E;2Uh+K+eu>|2*
zF8so%UuOX3{EyZgOw6S?&<+9z+W0@)fxm5+I<VK%B6+O-wjG264Q3l0kch26>O^du
zb}2Vek!?GCrx}%>KrjYOtc-i7bNKWKGF8Axg3%V!k?@Uo{X0L;#Eskl4YRB1;(2W)
zxrB;VEuU(;Z!Y;OU)HPhBm(Z(d+B${*NZf(ji0Hjd90p{Il1&<IJz^3b6#bpv59om
zt*3_ffM<=#nJOH!i65Kns-hIN$uY&+{*9e9_^~wozWt0NKE$l*+ir7~ru>38fJem$
zYZPkqQY4I45;JR*)LR-}Jv*QDl(0Q>IVaLX19N<oedZioa|IZV4MwGXZW*!~%xl0!
z9b4FL(Ao+&?9yuc)K1Ee^oRC(fK!3Q+)LtEkL=#5758CxW=ts4S@E-of-E2pZ{C~B
z6K-l={kuMbctxkqZuO}K@KZ`+0owRTD=Vpa<Z;euKaZ7gtW3@|*YdSSA-EOh0oUFz
zSn(wd|8@maUQ|F<&Ka4QCoJ5BM!EHJ+N=`SkL;Z0!>sC;+X%WGef8SM9d{`4SpiGs
z3zZ_b9;WH|D=&m8*-2F>4dD;Nw*FoS$Weh(mISa~DxRizZ@=R!4(Txa_2OpRJFd_t
z`@Z^e7TM41QrXniAVUDnmD06t;rdL*V7~ty?P4^98}$kun&?L^PUk+1)wRWoR+Mbl
zOa%#`T;8@|6|`@%v}+gZYji;7&;!{M&f`CZAC#I``Zp7)iqHC2f~Cyh8(#MGTgJiC
zc5MAgrRQMt)~k>;SwgHI-s_YZ`kAcYI}^pZXP^xqey*H|1L@*8=KX}13eU=mUk{cW
z1e`>uJ=Ue(G2{_yB2AL0+y;Tjn|xrRpKXbJ6~81vBJz+bGSOwI<8$A0&FG-zr{7bn
z&AW_{X=a(v7Y||FGPtI?_h*D~n*LdtyNaJWJTH@iUUH1C8-nR*kLw>O?Lj8MT}8T5
zf;*rNGq0AzpF0ZKi<^-eh)B4Rv2*H_aG8%>qv_mkfpwbD!K>)2Ti{K{XiM~n^9M4f
zeMBg1&-Npv9U20wJjx%momF;g`T&DlD*J<8q<aWSUwG8GWK*XPxE?P^wUJFW8y0?2
zt@m0y$i8Xkn`3WAE$3_>WFe8{Xc^va$9o#F_|waW`n{%6=&zxYd5^iN1UiA0e`ftc
zi48JIu=&`;)@l5Sm6p0{i<tLal<#i3dU7;>MGff21?X%0YF#^4HpQKI<L44I@7P&H
zs&^#3UQbPn;Y@Q3L=SXuXfsp@t-74%0a4%idSs@wuXJ8Cl!X#tY<!aalMCGqR^!-i
zM*{~Z|EsqtxZ3MTTi+S!I@V7qOV8upH!=o-hL)})Bbrjz-U-^!wb>GzT0$bIqG{c9
zXAenf-qpmzi~<(cgvq-JqU?L&?+3O`Qw36?_a|B_pPoGR1Qv|i{s?{)uJ^Zt)##h<
z8YITGmK+e;pC`M<u#zEVaBb1oIozqNr}eK+iAB+kMwIs5@?P`iyB~gK7Tw;@Wv-ro
zH~2<IW*ISN`dY3qcj@>78GUJSB3Y8Bq6p5j2mcsZZk;1?-b6QE6U2i`(m=+?v4w4y
z;N5ueEnlM&7C%!GQ0d&;X1&%Zc|NygS=}qz7fZ@^Q(V{QI@7nDnt42Al@o{txJ{C?
z*$dVxoh&B{u3+zmHX~Ji#OeR8BfEeq8x#x(1o!}=5YqqP)2f0X0az^P=%$5aBDA|}
z<xs`+-o`^>Cqfk=g<uD>C~*UsMOgqKfwA;%!mU@1`625qNb9~DiJYL%t{}F#n!ejA
Kf`EbF<M}UFD1Q<F


From c39c9aecf859095bfde87267e7eb9d9a4fd1682d Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 11 May 2017 17:16:53 -0700
Subject: [PATCH 219/387] Enable SerializableCoder to Serialize with Generic
 Types

A TypeToken that contains generics is not serializable. However, the
TypeDescriptor does not need to be transmitted via the serialized form,
so mark it as transient.
---
 .../src/main/resources/beam/findbugs-filter.xml        | 10 ++++++++++
 .../org/apache/beam/sdk/coders/SerializableCoder.java  |  5 ++++-
 .../apache/beam/sdk/coders/SerializableCoderTest.java  |  8 +++++++-
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
index 1db0e86d6afc..8ff0cb02eb3c 100644
--- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
@@ -73,6 +73,16 @@
     -->
   </Match>
 
+  <Match>
+    <Class name="org.apache.beam.sdk.coders.SerializableCoder"/>
+    <Field name="typeDescriptor"/>
+    <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED"/>
+    <!--
+    the field is used only in getEncodedTypeDescriptor, where it is restored if it is not present due to
+    serialization
+    -->
+  </Match>
+
   <Match>
     <Class name="org.apache.beam.sdk.io.jms.JmsRecord"/>
     <Field name="jmsDestination"/>
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
index 9aa8493e5e71..66918768818f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java
@@ -107,7 +107,7 @@ public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor,
   }
 
   private final Class<T> type;
-  private final TypeDescriptor<T> typeDescriptor;
+  private transient TypeDescriptor<T> typeDescriptor;
 
   protected SerializableCoder(Class<T> type, TypeDescriptor<T> typeDescriptor) {
     this.type = type;
@@ -166,6 +166,9 @@ public int hashCode() {
 
   @Override
   public TypeDescriptor<T> getEncodedTypeDescriptor() {
+    if (typeDescriptor == null) {
+      typeDescriptor = TypeDescriptor.of(type);
+    }
     return typeDescriptor;
   }
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
index adb665286eee..dd4f6ca345cd 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/SerializableCoderTest.java
@@ -125,11 +125,17 @@ public void testSerializableCoderConstruction() throws Exception {
     assertEquals(coder.getRecordType(), MyRecord.class);
     CoderProperties.coderSerializable(coder);
 
-
     SerializableCoder<?> decoded = SerializableUtils.clone(coder);
     assertThat(decoded.getRecordType(), Matchers.<Object>equalTo(MyRecord.class));
   }
 
+  @Test
+  public <T extends Serializable> void testSerializableCoderIsSerializableWithGenericTypeToken()
+  throws Exception {
+    SerializableCoder<T> coder = SerializableCoder.of(new TypeDescriptor<T>() {});
+    CoderProperties.coderSerializable(coder);
+  }
+
   @Test
   public void testNullEquals() {
     SerializableCoder<MyRecord> coder = SerializableCoder.of(MyRecord.class);

From 2f177820c4e72fe92dc47d595847ddecdae7afcd Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 11 May 2017 17:23:40 -0700
Subject: [PATCH 220/387] Rename filesink to filebasedsink

---
 sdks/python/apache_beam/io/__init__.py        |  2 +-
 sdks/python/apache_beam/io/avroio.py          |  4 +--
 .../io/{fileio.py => filebasedsink.py}        | 17 ++++------
 .../{fileio_test.py => filebasedsink_test.py} | 32 +++++++++----------
 sdks/python/apache_beam/io/gcp/gcsio.py       |  6 ++--
 sdks/python/apache_beam/io/iobase.py          | 12 ++++---
 sdks/python/apache_beam/io/textio.py          |  4 +--
 sdks/python/apache_beam/io/tfrecordio.py      |  6 ++--
 .../apache_beam/testing/pipeline_verifiers.py |  4 +--
 9 files changed, 43 insertions(+), 44 deletions(-)
 rename sdks/python/apache_beam/io/{fileio.py => filebasedsink.py} (96%)
 rename sdks/python/apache_beam/io/{fileio_test.py => filebasedsink_test.py} (93%)

diff --git a/sdks/python/apache_beam/io/__init__.py b/sdks/python/apache_beam/io/__init__.py
index 881ce684ec3c..6ea0efdf65f7 100644
--- a/sdks/python/apache_beam/io/__init__.py
+++ b/sdks/python/apache_beam/io/__init__.py
@@ -19,7 +19,7 @@
 
 # pylint: disable=wildcard-import
 from apache_beam.io.avroio import *
-from apache_beam.io.fileio import *
+from apache_beam.io.filebasedsink import *
 from apache_beam.io.iobase import Read
 from apache_beam.io.iobase import Sink
 from apache_beam.io.iobase import Write
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index 1c08c6848e6c..e02e1f7e893f 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -27,7 +27,7 @@
 
 import apache_beam as beam
 from apache_beam.io import filebasedsource
-from apache_beam.io import fileio
+from apache_beam.io import filebasedsink
 from apache_beam.io import iobase
 from apache_beam.io.filesystem import CompressionTypes
 from apache_beam.io.iobase import Read
@@ -335,7 +335,7 @@ def display_data(self):
     return {'sink_dd': self._sink}
 
 
-class _AvroSink(fileio.FileSink):
+class _AvroSink(filebasedsink.FileBasedSink):
   """A sink to avro files."""
 
   def __init__(self,
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/filebasedsink.py
similarity index 96%
rename from sdks/python/apache_beam/io/fileio.py
rename to sdks/python/apache_beam/io/filebasedsink.py
index aa1809318dad..76c09fc9b705 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/filebasedsink.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-"""File-based sources and sinks."""
+"""File-based sink."""
 
 from __future__ import absolute_import
 
@@ -40,7 +40,7 @@
 __all__ = ['FileBasedSink']
 
 
-class FileSink(iobase.Sink):
+class FileBasedSink(iobase.Sink):
   """A sink to a GCS or local files.
 
   To implement a file-based sink, extend this class and override
@@ -171,7 +171,7 @@ def open_writer(self, init_result, uid):
     file_name_suffix = self.file_name_suffix.get()
     suffix = (
         '.' + os.path.basename(file_path_prefix) + file_name_suffix)
-    return FileSinkWriter(self, os.path.join(init_result, uid) + suffix)
+    return FileBasedSinkWriter(self, os.path.join(init_result, uid) + suffix)
 
   @check_accessible(['file_path_prefix', 'file_name_suffix'])
   def finalize_write(self, init_result, writer_results):
@@ -179,7 +179,7 @@ def finalize_write(self, init_result, writer_results):
     file_name_suffix = self.file_name_suffix.get()
     writer_results = sorted(writer_results)
     num_shards = len(writer_results)
-    min_threads = min(num_shards, FileSink._MAX_RENAME_THREADS)
+    min_threads = min(num_shards, FileBasedSink._MAX_RENAME_THREADS)
     num_threads = max(1, min_threads)
 
     source_files = []
@@ -282,13 +282,8 @@ def __eq__(self, other):
     return type(self) == type(other) and self.__dict__ == other.__dict__
 
 
-# Using FileBasedSink for the public API to be symmetric with FileBasedSource.
-# TODO: move code from FileSink to here and delete that class.
-FileBasedSink = FileSink
-
-
-class FileSinkWriter(iobase.Writer):
-  """The writer for FileSink.
+class FileBasedSinkWriter(iobase.Writer):
+  """The writer for FileBasedSink.
   """
 
   def __init__(self, sink, temp_shard_path):
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/filebasedsink_test.py
similarity index 93%
rename from sdks/python/apache_beam/io/fileio_test.py
rename to sdks/python/apache_beam/io/filebasedsink_test.py
index b92b8beda7ff..1f6aeee56677 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/filebasedsink_test.py
@@ -30,7 +30,7 @@
 
 import apache_beam as beam
 from apache_beam.coders import coders
-from apache_beam.io import fileio
+from apache_beam.io import filebasedsink
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
@@ -70,12 +70,12 @@ def _create_temp_file(self, name='', suffix=''):
     return file_name
 
 
-class MyFileSink(fileio.FileSink):
+class MyFileBasedSink(filebasedsink.FileBasedSink):
 
   def open(self, temp_path):
     # TODO: Fix main session pickling.
-    # file_handle = super(MyFileSink, self).open(temp_path)
-    file_handle = fileio.FileSink.open(self, temp_path)
+    # file_handle = super(MyFileBasedSink, self).open(temp_path)
+    file_handle = filebasedsink.FileBasedSink.open(self, temp_path)
     file_handle.write('[start]')
     return file_handle
 
@@ -87,15 +87,15 @@ def write_encoded_record(self, file_handle, encoded_value):
   def close(self, file_handle):
     file_handle.write('[end]')
     # TODO: Fix main session pickling.
-    # file_handle = super(MyFileSink, self).close(file_handle)
-    file_handle = fileio.FileSink.close(self, file_handle)
+    # file_handle = super(MyFileBasedSink, self).close(file_handle)
+    file_handle = filebasedsink.FileBasedSink.close(self, file_handle)
 
 
-class TestFileSink(_TestCaseWithTempDirCleanUp):
+class TestFileBasedSink(_TestCaseWithTempDirCleanUp):
 
   def test_file_sink_writing(self):
-    temp_path = os.path.join(self._new_tempdir(), 'filesink')
-    sink = MyFileSink(
+    temp_path = os.path.join(self._new_tempdir(), 'FileBasedSink')
+    sink = MyFileBasedSink(
         temp_path, file_name_suffix='.output', coder=coders.ToStringCoder())
 
     # Manually invoke the generic Sink API.
@@ -128,7 +128,7 @@ def test_file_sink_writing(self):
 
   def test_file_sink_display_data(self):
     temp_path = os.path.join(self._new_tempdir(), 'display')
-    sink = MyFileSink(
+    sink = MyFileBasedSink(
         temp_path, file_name_suffix='.output', coder=coders.ToStringCoder())
     dd = DisplayData.create_from(sink)
     expected_items = [
@@ -143,7 +143,7 @@ def test_file_sink_display_data(self):
 
   def test_empty_write(self):
     temp_path = tempfile.NamedTemporaryFile().name
-    sink = MyFileSink(
+    sink = MyFileBasedSink(
         temp_path, file_name_suffix='.output', coder=coders.ToStringCoder()
     )
     p = TestPipeline()
@@ -155,7 +155,7 @@ def test_empty_write(self):
   def test_static_value_provider_empty_write(self):
     temp_path = StaticValueProvider(value_type=str,
                                     value=tempfile.NamedTemporaryFile().name)
-    sink = MyFileSink(
+    sink = MyFileBasedSink(
         temp_path,
         file_name_suffix=StaticValueProvider(value_type=str, value='.output'),
         coder=coders.ToStringCoder()
@@ -168,7 +168,7 @@ def test_static_value_provider_empty_write(self):
 
   def test_fixed_shard_write(self):
     temp_path = os.path.join(self._new_tempdir(), 'empty')
-    sink = MyFileSink(
+    sink = MyFileBasedSink(
         temp_path,
         file_name_suffix='.output',
         num_shards=3,
@@ -189,7 +189,7 @@ def test_fixed_shard_write(self):
   def run_temp_dir_check(self, no_dir_path, dir_path, no_dir_root_path,
                          dir_root_path, prefix, separator):
     def _get_temp_dir(file_path_prefix):
-      sink = MyFileSink(
+      sink = MyFileBasedSink(
           file_path_prefix, file_name_suffix='.output',
           coder=coders.ToStringCoder())
       return sink.initialize_write()
@@ -241,7 +241,7 @@ def _fake_unix_join(base, path):
 
   def test_file_sink_multi_shards(self):
     temp_path = os.path.join(self._new_tempdir(), 'multishard')
-    sink = MyFileSink(
+    sink = MyFileBasedSink(
         temp_path, file_name_suffix='.output', coder=coders.ToStringCoder())
 
     # Manually invoke the generic Sink API.
@@ -276,7 +276,7 @@ def test_file_sink_multi_shards(self):
 
   def test_file_sink_io_error(self):
     temp_path = os.path.join(self._new_tempdir(), 'ioerror')
-    sink = MyFileSink(
+    sink = MyFileBasedSink(
         temp_path, file_name_suffix='.output', coder=coders.ToStringCoder())
 
     # Manually invoke the generic Sink API.
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index 774ee549fc1f..7e21586381c2 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -257,9 +257,9 @@ def copy(self, src, dest):
       self.client.objects.Copy(request)
     except HttpError as http_error:
       if http_error.status_code == 404:
-        # This is a permanent error that should not be retried.  Note that
-        # FileSink.finalize_write expects an IOError when the source file does
-        # not exist.
+        # This is a permanent error that should not be retried. Note that
+        # FileBasedSink.finalize_write expects an IOError when the source
+        # file does not exist.
         raise GcsIOError(errno.ENOENT, 'Source file not found: %s' % src)
       raise
 
diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py
index a80b12f72cfb..7e40d833e472 100644
--- a/sdks/python/apache_beam/io/iobase.py
+++ b/sdks/python/apache_beam/io/iobase.py
@@ -562,7 +562,9 @@ def set_split_points_unclaimed_callback(self, callback):
 
 
 class Sink(HasDisplayData):
-  """A resource that can be written to using the ``beam.io.Write`` transform.
+  """This class is deprecated, no backwards-compatibility guarantees.
+
+  A resource that can be written to using the ``beam.io.Write`` transform.
 
   Here ``beam`` stands for Apache Beam Python code imported in following manner.
   ``import apache_beam as beam``.
@@ -594,8 +596,8 @@ class Sink(HasDisplayData):
   single record from the bundle and ``close()`` which is called once
   at the end of writing a bundle.
 
-  See also ``apache_beam.io.fileio.FileSink`` which provides a simpler API
-  for writing sinks that produce files.
+  See also ``apache_beam.io.filebasedsink.FileBasedSink`` which provides a
+  simpler API for writing sinks that produce files.
 
   **Execution of the Write transform**
 
@@ -759,7 +761,9 @@ def finalize_write(self, init_result, writer_results):
 
 
 class Writer(object):
-  """Writes a bundle of elements from a ``PCollection`` to a sink.
+  """This class is deprecated, no backwards-compatibility guarantees.
+
+  Writes a bundle of elements from a ``PCollection`` to a sink.
 
   A Writer  ``iobase.Writer.write()`` writes and elements to the sink while
   ``iobase.Writer.close()`` is called after all elements in the bundle have been
diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py
index d43f4fca9ee7..eeefaf6fd94e 100644
--- a/sdks/python/apache_beam/io/textio.py
+++ b/sdks/python/apache_beam/io/textio.py
@@ -23,7 +23,7 @@
 
 from apache_beam.coders import coders
 from apache_beam.io import filebasedsource
-from apache_beam.io import fileio
+from apache_beam.io import filebasedsink
 from apache_beam.io import iobase
 from apache_beam.io.filesystem import CompressionTypes
 from apache_beam.io.iobase import Read
@@ -262,7 +262,7 @@ def _read_record(self, file_to_read, read_buffer):
               sep_bounds[1] - record_start_position_in_buffer)
 
 
-class _TextSink(fileio.FileSink):
+class _TextSink(filebasedsink.FileBasedSink):
   """A sink to a GCS or local text file or files."""
 
   def __init__(self,
diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py
index 2f7f4dca85f8..a8cd1ce25cc5 100644
--- a/sdks/python/apache_beam/io/tfrecordio.py
+++ b/sdks/python/apache_beam/io/tfrecordio.py
@@ -23,7 +23,7 @@
 
 from apache_beam import coders
 from apache_beam.io import filebasedsource
-from apache_beam.io import fileio
+from apache_beam.io import filebasedsink
 from apache_beam.io.filesystem import CompressionTypes
 from apache_beam.io.iobase import Read
 from apache_beam.io.iobase import Write
@@ -210,7 +210,7 @@ def expand(self, pvalue):
     return pvalue.pipeline | Read(self._source)
 
 
-class _TFRecordSink(fileio.FileSink):
+class _TFRecordSink(filebasedsink.FileBasedSink):
   """Sink for writing TFRecords files.
 
   For detailed TFRecord format description see:
@@ -242,7 +242,7 @@ def __init__(self,
                coder=coders.BytesCoder(),
                file_name_suffix='',
                num_shards=0,
-               shard_name_template=fileio.DEFAULT_SHARD_NAME_TEMPLATE,
+               shard_name_template=filebasedsink.DEFAULT_SHARD_NAME_TEMPLATE,
                compression_type=CompressionTypes.AUTO,
                **kwargs):
     """Initialize WriteToTFRecord transform.
diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers.py b/sdks/python/apache_beam/testing/pipeline_verifiers.py
index a08eb5409cdd..883343acdab6 100644
--- a/sdks/python/apache_beam/testing/pipeline_verifiers.py
+++ b/sdks/python/apache_beam/testing/pipeline_verifiers.py
@@ -81,8 +81,8 @@ def retry_on_io_error_and_server_error(exception):
 class FileChecksumMatcher(BaseMatcher):
   """Matcher that verifies file(s) content by comparing file checksum.
 
-  Use apache_beam.io.fileio to fetch file(s) from given path. File checksum
-  is a hash string computed from content of file(s).
+  Use apache_beam.io.filebasedsink to fetch file(s) from given path.
+  File checksum is a hash string computed from content of file(s).
   """
 
   def __init__(self, file_path, expected_checksum, sleep_secs=None):

From db7d71aec5d6c4b7a15474c7ef9db28460fb0043 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 11 May 2017 16:17:54 -0700
Subject: [PATCH 221/387] Mark More values methods Internal

PCollectionTuple#ofPrimitiveOutputsInternal is internal.

TaggedPValue is internal.
---
 .../java/org/apache/beam/sdk/values/PCollectionTuple.java   | 6 +++++-
 .../main/java/org/apache/beam/sdk/values/TaggedPValue.java  | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
index d1bb6d7b5667..793994f102e9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Objects;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
@@ -189,11 +190,14 @@ public <OutputT extends POutput> OutputT apply(
   }
 
   /**
-   * Returns a {@link PCollectionTuple} with each of the given tags mapping to a new
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Returns a {@link PCollectionTuple} with each of the given tags mapping to a new
    * output {@link PCollection}.
    *
    * <p>For use by primitive transformations only.
    */
+  @Internal
   public static PCollectionTuple ofPrimitiveOutputsInternal(
       Pipeline pipeline,
       TupleTagList outputTags,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java
index 3b4d5994e652..c1e0209bf9a2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java
@@ -21,11 +21,15 @@
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.Iterables;
+import org.apache.beam.sdk.annotations.Internal;
 
 /**
- * A (TupleTag, PValue) pair used in the expansion of a {@link PInput} or {@link POutput}.
+ * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+ *
+ * <p>A (TupleTag, PValue) pair used in the expansion of a {@link PInput} or {@link POutput}.
  */
 @AutoValue
+@Internal
 public abstract class TaggedPValue {
   public static TaggedPValue of(TupleTag<?> tag, PValue value) {
     return new AutoValue_TaggedPValue(tag, value);

From 5228f7d1f48bf39c0f97c28ab64064089e5f8df9 Mon Sep 17 00:00:00 2001
From: Borisa Zivkovic <borisa.zivkovic@huawei.com>
Date: Fri, 12 May 2017 14:03:02 +0100
Subject: [PATCH 222/387] [BEAM-2153] Move connection management in
 JmsIO.write() to setup/teardown methods

---
 .../java/org/apache/beam/sdk/io/jms/JmsIO.java  | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
index 4493e56d3fee..b8355ad4a30f 100644
--- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
+++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java
@@ -673,8 +673,8 @@ public WriterFn(Write spec) {
         this.spec = spec;
       }
 
-      @StartBundle
-      public void startBundle() throws Exception {
+      @Setup
+      public void setup() throws Exception {
         if (producer == null) {
           if (spec.getUsername() != null) {
             this.connection =
@@ -699,17 +699,12 @@ public void startBundle() throws Exception {
       @ProcessElement
       public void processElement(ProcessContext ctx) throws Exception {
         String value = ctx.element();
-        try {
-          TextMessage message = session.createTextMessage(value);
-          producer.send(message);
-        } catch (Exception t) {
-          finishBundle();
-          throw t;
-        }
+        TextMessage message = session.createTextMessage(value);
+        producer.send(message);
       }
 
-      @FinishBundle
-      public void finishBundle() throws Exception {
+      @Teardown
+      public void teardown() throws Exception {
         producer.close();
         producer = null;
         session.close();

From e8d829817e3f7e07884a91737b61bfa5133d5724 Mon Sep 17 00:00:00 2001
From: Borisa Zivkovic <borisa.zivkovic@huawei.com>
Date: Fri, 12 May 2017 08:44:20 +0100
Subject: [PATCH 223/387] BigtableIO should use AutoValue for read and write

---
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java  | 237 +++++++++---------
 1 file changed, 114 insertions(+), 123 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index bde7ea500c34..22e9f3611ec7 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -21,6 +21,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
+import com.google.auto.value.AutoValue;
 import com.google.bigtable.v2.MutateRowResponse;
 import com.google.bigtable.v2.Mutation;
 import com.google.bigtable.v2.Row;
@@ -164,7 +165,8 @@ public class BigtableIO {
    */
   @Experimental
   public static Read read() {
-    return new Read(null, "", ByteKeyRange.ALL_KEYS, null, null);
+    return new AutoValue_BigtableIO_Read.Builder().setKeyRange(ByteKeyRange.ALL_KEYS).setTableId("")
+        .build();
   }
 
   /**
@@ -176,7 +178,7 @@ public static Read read() {
    */
   @Experimental
   public static Write write() {
-    return new Write(null, "", null);
+    return new AutoValue_BigtableIO_Write.Builder().setTableId("").build();
   }
 
   /**
@@ -186,7 +188,46 @@ public static Write write() {
    * @see BigtableIO
    */
   @Experimental
-  public static class Read extends PTransform<PBegin, PCollection<Row>> {
+  @AutoValue
+  public abstract static class Read extends PTransform<PBegin, PCollection<Row>> {
+
+    @Nullable
+    abstract RowFilter getRowFilter();
+
+    /** Returns the range of keys that will be read from the table. */
+    @Nullable
+    public abstract ByteKeyRange getKeyRange();
+
+    /** Returns the table being read from. */
+    @Nullable
+    public abstract String getTableId();
+
+    @Nullable
+    abstract BigtableService getBigtableService();
+
+
+    /** Returns the Google Cloud Bigtable instance being read from, and other parameters. */
+    @Nullable
+    public abstract BigtableOptions getBigtableOptions();
+
+    abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    abstract static class Builder {
+
+      abstract Builder setRowFilter(RowFilter filter);
+
+      abstract Builder setKeyRange(ByteKeyRange keyRange);
+
+      abstract Builder setTableId(String tableId);
+
+      abstract Builder setBigtableOptions(BigtableOptions options);
+
+      abstract Builder setBigtableService(BigtableService bigtableService);
+
+      abstract Read build();
+    }
+
     /**
      * Returns a new {@link BigtableIO.Read} that will read from the Cloud Bigtable instance
      * indicated by the given options, and using any other specified customizations.
@@ -217,7 +258,7 @@ public Read withBigtableOptions(BigtableOptions.Builder optionsBuilder) {
       BigtableOptions optionsWithAgent =
           clonedBuilder.setUserAgent(getBeamSdkPartOfUserAgent()).build();
 
-      return new Read(optionsWithAgent, tableId, keyRange, filter, bigtableService);
+      return toBuilder().setBigtableOptions(optionsWithAgent).build();
     }
 
     /**
@@ -228,7 +269,7 @@ public Read withBigtableOptions(BigtableOptions.Builder optionsBuilder) {
      */
     public Read withRowFilter(RowFilter filter) {
       checkNotNull(filter, "filter");
-      return new Read(options, tableId, keyRange, filter, bigtableService);
+      return toBuilder().setRowFilter(filter).build();
     }
 
     /**
@@ -238,7 +279,7 @@ public Read withRowFilter(RowFilter filter) {
      */
     public Read withKeyRange(ByteKeyRange keyRange) {
       checkNotNull(keyRange, "keyRange");
-      return new Read(options, tableId, keyRange, filter, bigtableService);
+      return toBuilder().setKeyRange(keyRange).build();
     }
 
     /**
@@ -248,29 +289,7 @@ public Read withKeyRange(ByteKeyRange keyRange) {
      */
     public Read withTableId(String tableId) {
       checkNotNull(tableId, "tableId");
-      return new Read(options, tableId, keyRange, filter, bigtableService);
-    }
-
-    /**
-     * Returns the Google Cloud Bigtable instance being read from, and other parameters.
-     */
-    public BigtableOptions getBigtableOptions() {
-      return options;
-    }
-
-    /**
-     * Returns the range of keys that will be read from the table. By default, returns
-     * {@link ByteKeyRange#ALL_KEYS} to scan the entire table.
-     */
-    public ByteKeyRange getKeyRange() {
-      return keyRange;
-    }
-
-    /**
-     * Returns the table being read from.
-     */
-    public String getTableId() {
-      return tableId;
+      return toBuilder().setTableId(tableId).build();
     }
 
     @Override
@@ -281,21 +300,21 @@ public PCollection<Row> expand(PBegin input) {
             public BigtableService apply(PipelineOptions options) {
               return getBigtableService(options);
             }
-          }, tableId, filter, keyRange, null);
+          }, getTableId(), getRowFilter(), getKeyRange(), null);
       return input.getPipeline().apply(org.apache.beam.sdk.io.Read.from(source));
     }
 
     @Override
     public void validate(PipelineOptions options) {
-      checkArgument(this.options != null, "BigtableOptions not specified");
-      checkArgument(!tableId.isEmpty(), "Table ID not specified");
+      checkArgument(getBigtableOptions() != null, "BigtableOptions not specified");
+      checkArgument(getTableId() != null && !getTableId().isEmpty(), "Table ID not specified");
       try {
         checkArgument(
-            getBigtableService(options).tableExists(tableId),
+            getBigtableService(options).tableExists(getTableId()),
             "Table %s does not exist",
-            tableId);
+            getTableId());
       } catch (IOException e) {
-        LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e);
+        LOG.warn("Error checking whether table {} exists; proceeding.", getTableId(), e);
       }
     }
 
@@ -303,19 +322,19 @@ public void validate(PipelineOptions options) {
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
 
-      builder.add(DisplayData.item("tableId", tableId)
+      builder.add(DisplayData.item("tableId", getTableId())
         .withLabel("Table ID"));
 
-      if (options != null) {
-        builder.add(DisplayData.item("bigtableOptions", options.toString())
+      if (getBigtableOptions() != null) {
+        builder.add(DisplayData.item("bigtableOptions", getBigtableOptions().toString())
           .withLabel("Bigtable Options"));
       }
 
       builder.addIfNotDefault(
-          DisplayData.item("keyRange", keyRange.toString()), ByteKeyRange.ALL_KEYS.toString());
+          DisplayData.item("keyRange", getKeyRange().toString()), ByteKeyRange.ALL_KEYS.toString());
 
-      if (filter != null) {
-        builder.add(DisplayData.item("rowFilter", filter.toString())
+      if (getRowFilter() != null) {
+        builder.add(DisplayData.item("rowFilter", getRowFilter().toString())
           .withLabel("Table Row Filter"));
       }
     }
@@ -323,38 +342,13 @@ public void populateDisplayData(DisplayData.Builder builder) {
     @Override
     public String toString() {
       return MoreObjects.toStringHelper(Read.class)
-          .add("options", options)
-          .add("tableId", tableId)
-          .add("keyRange", keyRange)
-          .add("filter", filter)
+          .add("options", getBigtableOptions())
+          .add("tableId", getTableId())
+          .add("keyRange", getKeyRange())
+          .add("filter", getRowFilter())
           .toString();
     }
 
-    /////////////////////////////////////////////////////////////////////////////////////////
-    /**
-     * Used to define the Cloud Bigtable instance and any options for the networking layer.
-     * Cannot actually be {@code null} at validation time, but may start out {@code null} while
-     * source is being built.
-     */
-    @Nullable private final BigtableOptions options;
-    private final String tableId;
-    private final ByteKeyRange keyRange;
-    @Nullable private final RowFilter filter;
-    @Nullable private final BigtableService bigtableService;
-
-    private Read(
-        @Nullable BigtableOptions options,
-        String tableId,
-        ByteKeyRange keyRange,
-        @Nullable RowFilter filter,
-        @Nullable BigtableService bigtableService) {
-      this.options = options;
-      this.tableId = checkNotNull(tableId, "tableId");
-      this.keyRange = checkNotNull(keyRange, "keyRange");
-      this.filter = filter;
-      this.bigtableService = bigtableService;
-    }
-
     /**
      * Returns a new {@link BigtableIO.Read} that will read using the given Cloud Bigtable
      * service implementation.
@@ -365,7 +359,7 @@ private Read(
      */
     Read withBigtableService(BigtableService bigtableService) {
       checkNotNull(bigtableService, "bigtableService");
-      return new Read(options, tableId, keyRange, filter, bigtableService);
+      return toBuilder().setBigtableService(bigtableService).build();
     }
 
     /**
@@ -378,11 +372,12 @@ Read withBigtableService(BigtableService bigtableService) {
      */
     @VisibleForTesting
     BigtableService getBigtableService(PipelineOptions pipelineOptions) {
-      if (bigtableService != null) {
-        return bigtableService;
+      if (getBigtableService() != null) {
+        return getBigtableService();
       }
-      BigtableOptions.Builder clonedOptions = options.toBuilder();
-      if (options.getCredentialOptions().getCredentialType() == CredentialType.DefaultCredentials) {
+      BigtableOptions.Builder clonedOptions = getBigtableOptions().toBuilder();
+      if (getBigtableOptions().getCredentialOptions()
+          .getCredentialType() == CredentialType.DefaultCredentials) {
         clonedOptions.setCredentialOptions(
             CredentialOptions.credential(
                 pipelineOptions.as(GcpOptions.class).getGcpCredential()));
@@ -398,24 +393,33 @@ BigtableService getBigtableService(PipelineOptions pipelineOptions) {
    * @see BigtableIO
    */
   @Experimental
-  public static class Write
+  @AutoValue
+  public abstract static class Write
       extends PTransform<PCollection<KV<ByteString, Iterable<Mutation>>>, PDone> {
-    /**
-     * Used to define the Cloud Bigtable instance and any options for the networking layer.
-     * Cannot actually be {@code null} at validation time, but may start out {@code null} while
-     * source is being built.
-     */
-    @Nullable private final BigtableOptions options;
-    private final String tableId;
-    @Nullable private final BigtableService bigtableService;
 
-    private Write(
-        @Nullable BigtableOptions options,
-        String tableId,
-        @Nullable BigtableService bigtableService) {
-      this.options = options;
-      this.tableId = checkNotNull(tableId, "tableId");
-      this.bigtableService = bigtableService;
+    /** Returns the table being written to. */
+    @Nullable
+    abstract String getTableId();
+
+    @Nullable
+    abstract BigtableService getBigtableService();
+
+    /** Returns the Google Cloud Bigtable instance being written to, and other parameters. */
+    @Nullable
+    public abstract BigtableOptions getBigtableOptions();
+
+    abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    abstract static class Builder {
+
+      abstract Builder setTableId(String tableId);
+
+      abstract Builder setBigtableOptions(BigtableOptions options);
+
+      abstract Builder setBigtableService(BigtableService bigtableService);
+
+      abstract Write build();
     }
 
     /**
@@ -452,7 +456,7 @@ public Write withBigtableOptions(BigtableOptions.Builder optionsBuilder) {
           .setUseCachedDataPool(true);
       BigtableOptions optionsWithAgent =
           clonedBuilder.setUserAgent(getBeamSdkPartOfUserAgent()).build();
-      return new Write(optionsWithAgent, tableId, bigtableService);
+      return toBuilder().setBigtableOptions(optionsWithAgent).build();
     }
 
     /**
@@ -462,26 +466,12 @@ public Write withBigtableOptions(BigtableOptions.Builder optionsBuilder) {
      */
     public Write withTableId(String tableId) {
       checkNotNull(tableId, "tableId");
-      return new Write(options, tableId, bigtableService);
-    }
-
-    /**
-     * Returns the Google Cloud Bigtable instance being written to, and other parameters.
-     */
-    public BigtableOptions getBigtableOptions() {
-      return options;
-    }
-
-    /**
-     * Returns the table being written to.
-     */
-    public String getTableId() {
-      return tableId;
+      return toBuilder().setTableId(tableId).build();
     }
 
     @Override
     public PDone expand(PCollection<KV<ByteString, Iterable<Mutation>>> input) {
-      input.apply(ParDo.of(new BigtableWriterFn(tableId,
+      input.apply(ParDo.of(new BigtableWriterFn(getTableId(),
           new SerializableFunction<PipelineOptions, BigtableService>() {
         @Override
         public BigtableService apply(PipelineOptions options) {
@@ -493,15 +483,15 @@ public BigtableService apply(PipelineOptions options) {
 
     @Override
     public void validate(PipelineOptions options) {
-      checkArgument(this.options != null, "BigtableOptions not specified");
-      checkArgument(!tableId.isEmpty(), "Table ID not specified");
+      checkArgument(getBigtableOptions() != null, "BigtableOptions not specified");
+      checkArgument(getTableId() != null && !getTableId().isEmpty(), "Table ID not specified");
       try {
         checkArgument(
-            getBigtableService(options).tableExists(tableId),
+            getBigtableService(options).tableExists(getTableId()),
             "Table %s does not exist",
-            tableId);
+            getTableId());
       } catch (IOException e) {
-        LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e);
+        LOG.warn("Error checking whether table {} exists; proceeding.", getTableId(), e);
       }
     }
 
@@ -515,18 +505,18 @@ public void validate(PipelineOptions options) {
      */
     Write withBigtableService(BigtableService bigtableService) {
       checkNotNull(bigtableService, "bigtableService");
-      return new Write(options, tableId, bigtableService);
+      return toBuilder().setBigtableService(bigtableService).build();
     }
 
     @Override
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
 
-      builder.add(DisplayData.item("tableId", tableId)
+      builder.add(DisplayData.item("tableId", getTableId())
         .withLabel("Table ID"));
 
-      if (options != null) {
-        builder.add(DisplayData.item("bigtableOptions", options.toString())
+      if (getBigtableOptions() != null) {
+        builder.add(DisplayData.item("bigtableOptions", getBigtableOptions().toString())
           .withLabel("Bigtable Options"));
       }
     }
@@ -534,8 +524,8 @@ public void populateDisplayData(DisplayData.Builder builder) {
     @Override
     public String toString() {
       return MoreObjects.toStringHelper(Write.class)
-          .add("options", options)
-          .add("tableId", tableId)
+          .add("options", getBigtableOptions())
+          .add("tableId", getTableId())
           .toString();
     }
 
@@ -549,11 +539,12 @@ public String toString() {
      */
     @VisibleForTesting
     BigtableService getBigtableService(PipelineOptions pipelineOptions) {
-      if (bigtableService != null) {
-        return bigtableService;
+      if (getBigtableService() != null) {
+        return getBigtableService();
       }
-      BigtableOptions.Builder clonedOptions = options.toBuilder();
-      if (options.getCredentialOptions().getCredentialType() == CredentialType.DefaultCredentials) {
+      BigtableOptions.Builder clonedOptions = getBigtableOptions().toBuilder();
+      if (getBigtableOptions().getCredentialOptions()
+          .getCredentialType() == CredentialType.DefaultCredentials) {
         clonedOptions.setCredentialOptions(
             CredentialOptions.credential(
                 pipelineOptions.as(GcpOptions.class).getGcpCredential()));

From 6a90b9d164d98d916d0e0af2be39a740ce769076 Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Fri, 12 May 2017 16:02:54 +0300
Subject: [PATCH 224/387] [BEAM-2277] Fix URI_SCHEME_PATTERN in FileSystems

---
 .../core/src/main/java/org/apache/beam/sdk/io/FileSystems.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index 2e11177b3184..08e0deff7732 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -67,7 +67,7 @@ public class FileSystems {
 
   public static final String DEFAULT_SCHEME = "default";
   private static final Pattern URI_SCHEME_PATTERN = Pattern.compile(
-      "(?<scheme>[a-zA-Z][-a-zA-Z0-9+.]*)://.*");
+      "(?<scheme>[a-zA-Z][-a-zA-Z0-9+.]*):/.*");
 
   private static final AtomicReference<Map<String, FileSystem>> SCHEME_TO_FILESYSTEM =
       new AtomicReference<Map<String, FileSystem>>(

From abe62feb16f6eb59e294c37abf408f84c9280616 Mon Sep 17 00:00:00 2001
From: Aviem Zur <aviemzur@gmail.com>
Date: Fri, 12 May 2017 22:18:02 +0300
Subject: [PATCH 225/387] [BEAM-2279] Add HDFS support to Spark runner profiles
 in archetypes and examples

---
 examples/java/pom.xml                                        | 5 +++++
 examples/java8/pom.xml                                       | 5 +++++
 .../src/main/resources/archetype-resources/pom.xml           | 5 +++++
 .../examples/src/main/resources/archetype-resources/pom.xml  | 5 +++++
 4 files changed, 20 insertions(+)

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 8899f567bbf9..701e4fe76cce 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -108,6 +108,11 @@
           <artifactId>beam-runners-spark</artifactId>
           <scope>runtime</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.beam</groupId>
+          <artifactId>beam-sdks-java-io-hadoop-file-system</artifactId>
+          <scope>runtime</scope>
+        </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-streaming_2.10</artifactId>
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 9337e8910baa..56295a4ece78 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -108,6 +108,11 @@
           <artifactId>beam-runners-spark</artifactId>
           <scope>runtime</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.beam</groupId>
+          <artifactId>beam-sdks-java-io-hadoop-file-system</artifactId>
+          <scope>runtime</scope>
+        </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-streaming_2.10</artifactId>
diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 3c9c48020018..05826baf6ccb 100644
--- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -215,6 +215,11 @@
           <version>${beam.version}</version>
           <scope>runtime</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.beam</groupId>
+          <artifactId>beam-sdks-java-io-hadoop-file-system</artifactId>
+          <scope>runtime</scope>
+        </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-streaming_2.10</artifactId>
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index 00f69ffa84b6..fb6e8b18d633 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -215,6 +215,11 @@
           <version>${beam.version}</version>
           <scope>runtime</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.beam</groupId>
+          <artifactId>beam-sdks-java-io-hadoop-file-system</artifactId>
+          <scope>runtime</scope>
+        </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-streaming_2.10</artifactId>

From fbb0de129d881b6b6a1ae37fc8d75075c9d8df86 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 12 May 2017 11:13:00 -0700
Subject: [PATCH 226/387] Remove '/' entirely from determining FileSystem
 scheme

---
 .../src/main/java/org/apache/beam/sdk/io/FileSystems.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index 08e0deff7732..4341fab65450 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -66,8 +66,8 @@
 public class FileSystems {
 
   public static final String DEFAULT_SCHEME = "default";
-  private static final Pattern URI_SCHEME_PATTERN = Pattern.compile(
-      "(?<scheme>[a-zA-Z][-a-zA-Z0-9+.]*):/.*");
+  private static final Pattern FILE_SCHEME_PATTERN =
+      Pattern.compile("(?<scheme>[a-zA-Z][-a-zA-Z0-9+.]*):.*");
 
   private static final AtomicReference<Map<String, FileSystem>> SCHEME_TO_FILESYSTEM =
       new AtomicReference<Map<String, FileSystem>>(
@@ -416,7 +416,7 @@ private static String parseScheme(String spec) {
     // from their use in the URI spec. ('*' is not reserved).
     // Here, we just need the scheme, which is so circumscribed as to be
     // very easy to extract with a regex.
-    Matcher matcher = URI_SCHEME_PATTERN.matcher(spec);
+    Matcher matcher = FILE_SCHEME_PATTERN.matcher(spec);
 
     if (!matcher.matches()) {
       return "file";

From a6a5ff7be387ef295fc7f921de36a3ea77327bc1 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 12 May 2017 09:20:34 -0700
Subject: [PATCH 227/387] [BEAM-2277] Add ResourceIdTester and test existing
 ResourceId implementations

A first cut at some of the parts of the ResourceId spec.
---
 .../beam/sdk/io/LocalResourceIdTest.java      |   6 +
 .../beam/sdk/io/fs/ResourceIdTester.java      | 151 ++++++++++++++++++
 .../google-cloud-platform-core/pom.xml        |   6 +
 .../gcp/storage/GcsResourceIdTest.java        |   9 ++
 sdks/java/io/hadoop-file-system/pom.xml       |  13 ++
 .../sdk/io/hdfs/HadoopResourceIdTest.java     |  63 ++++++++
 6 files changed, 248 insertions(+)
 create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java
 create mode 100644 sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java

diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
index 7ea85cfb079d..e1ca30382c7a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
@@ -31,6 +31,7 @@
 import java.nio.file.Paths;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.fs.ResourceIdTester;
 import org.apache.commons.lang3.SystemUtils;
 import org.junit.Rule;
 import org.junit.Test;
@@ -259,6 +260,11 @@ public void testGetFilename() throws Exception {
         "xyz.txt");
   }
 
+  @Test
+  public void testResourceIdTester() throws Exception {
+    ResourceIdTester.runResourceIdBattery(toResourceIdentifier("/tmp/foo/"));
+  }
+
   private LocalResourceId toResourceIdentifier(String str) throws Exception {
     boolean isDirectory;
     if (SystemUtils.IS_OS_WINDOWS) {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java
new file mode 100644
index 000000000000..fe50adac6983
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.fs;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY;
+import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.testing.EqualsTester;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.io.FileSystems;
+
+/**
+ * A utility to test {@link ResourceId} implementations.
+ */
+@Experimental(Kind.FILESYSTEM)
+public final class ResourceIdTester {
+  /**
+   * Enforces that the {@link ResourceId} implementation of {@code baseDirectory} meets the
+   * {@link ResourceId} spec.
+   */
+  public static void runResourceIdBattery(ResourceId baseDirectory) {
+    checkArgument(
+        baseDirectory.isDirectory(), "baseDirectory %s is not a directory", baseDirectory);
+
+    List<ResourceId> allResourceIds = new ArrayList<>();
+    allResourceIds.add(baseDirectory);
+
+    // Validate that individual resources meet the fairly restrictive spec we have.
+    validateResourceIds(allResourceIds);
+
+    // Validate operations with resolving child resources.
+    validateResolvingIds(baseDirectory, allResourceIds);
+
+    // Validate safeguards against resolving bad paths.
+    validateFailureResolvingIds(baseDirectory);
+  }
+
+  private static void validateResolvingIds(
+      ResourceId baseDirectory, List<ResourceId> allResourceIds) {
+    ResourceId file1 = baseDirectory.resolve("child1", RESOLVE_FILE);
+    ResourceId file2 = baseDirectory.resolve("child2", RESOLVE_FILE);
+    ResourceId file2a = baseDirectory.resolve("child2", RESOLVE_FILE);
+    allResourceIds.add(file1);
+    allResourceIds.add(file2);
+    assertFalse("Resolved file isDirectory()", file1.isDirectory());
+    assertFalse("Resolved file isDirectory()", file2.isDirectory());
+    assertFalse("Resolved file isDirectory()", file2a.isDirectory());
+
+    ResourceId dir1 = baseDirectory.resolve("child1", RESOLVE_DIRECTORY);
+    ResourceId dir2 = baseDirectory.resolve("child2", RESOLVE_DIRECTORY);
+    ResourceId dir2a = baseDirectory.resolve("child2", RESOLVE_DIRECTORY);
+    assertTrue("Resolved directory isDirectory()", dir1.isDirectory());
+    assertTrue("Resolved directory isDirectory()", dir2.isDirectory());
+    assertTrue("Resolved directory isDirectory()", dir2a.isDirectory());
+    allResourceIds.add(dir1);
+    allResourceIds.add(dir2);
+
+    // ResourceIds in equality groups.
+    new EqualsTester()
+        .addEqualityGroup(file1)
+        .addEqualityGroup(file2, file2a)
+        .addEqualityGroup(dir1, dir1.getCurrentDirectory())
+        .addEqualityGroup(dir2, dir2a, dir2.getCurrentDirectory())
+        .addEqualityGroup(baseDirectory, file1.getCurrentDirectory(), file2.getCurrentDirectory())
+        .testEquals();
+
+    // ResourceId toString() in equality groups.
+    new EqualsTester()
+        .addEqualityGroup(file1.toString())
+        .addEqualityGroup(file2.toString(), file2a.toString())
+        .addEqualityGroup(dir1.toString(), dir1.getCurrentDirectory().toString())
+        .addEqualityGroup(dir2.toString(), dir2a.toString(), dir2.getCurrentDirectory().toString())
+        .addEqualityGroup(
+            baseDirectory.toString(),
+            file1.getCurrentDirectory().toString(),
+            file2.getCurrentDirectory().toString())
+        .testEquals();
+
+    // TODO: test resolving strings that need to be escaped.
+    //   Possible spec: https://tools.ietf.org/html/rfc3986#section-2
+    //   May need options to be filesystem-independent, e.g., if filesystems ban certain chars.
+  }
+
+  private static void validateFailureResolvingIds(ResourceId baseDirectory) {
+    try {
+      ResourceId badFile = baseDirectory.resolve("file/", RESOLVE_FILE);
+      fail(String.format("Resolving badFile %s should have failed", badFile));
+    } catch (Throwable t) {
+      // expected
+    }
+
+    ResourceId file = baseDirectory.resolve("file", RESOLVE_FILE);
+    try {
+      baseDirectory.resolve("file2", RESOLVE_FILE);
+      fail(String.format("Should not be able to resolve against file resource %s", file));
+    } catch (Throwable t) {
+      // expected
+    }
+  }
+
+  private static void validateResourceIds(List<ResourceId> resourceIds) {
+    for (ResourceId resourceId : resourceIds) {
+      // ResourceIds should equal themselves.
+      assertThat("ResourceId equal to itself", resourceId, equalTo(resourceId));
+
+      // ResourceIds should be clonable via FileSystems#matchNewResource.
+      ResourceId cloned;
+      if (resourceId.isDirectory()) {
+        cloned = FileSystems.matchNewResource(resourceId.toString(), true /* isDirectory */);
+      } else {
+        cloned = FileSystems.matchNewResource(resourceId.toString(), false /* isDirectory */);
+      }
+      assertThat(
+          "ResourceId equals clone of itself", cloned, equalTo(resourceId));
+      // .. and clones have consistent toString.
+      assertThat(
+          "ResourceId toString consistency", cloned.toString(), equalTo(resourceId.toString()));
+      // .. and have consistent isDirectory.
+      assertThat(
+          "ResourceId isDirectory consistency",
+          cloned.isDirectory(),
+          equalTo(resourceId.isDirectory()));
+    }
+  }
+
+  private ResourceIdTester() {} // prevent instantiation
+}
diff --git a/sdks/java/extensions/google-cloud-platform-core/pom.xml b/sdks/java/extensions/google-cloud-platform-core/pom.xml
index a1baea1e3289..e4e951b20f54 100644
--- a/sdks/java/extensions/google-cloud-platform-core/pom.xml
+++ b/sdks/java/extensions/google-cloud-platform-core/pom.xml
@@ -151,6 +151,12 @@
     </dependency>
 
     <!-- test dependencies -->
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava-testlib</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-sdks-java-core</artifactId>
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
index b2456106994d..2a6750185f12 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
@@ -22,8 +22,11 @@
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.fs.ResourceIdTester;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.junit.Rule;
 import org.junit.Test;
@@ -163,6 +166,12 @@ public void testGetFilename() throws Exception {
         "xyz.txt");
   }
 
+  @Test
+  public void testResourceIdTester() throws Exception {
+    FileSystems.setDefaultConfigInWorkers(TestPipeline.testingPipelineOptions());
+    ResourceIdTester.runResourceIdBattery(toResourceIdentifier("gs://bucket/foo/"));
+  }
+
   private GcsResourceId toResourceIdentifier(String str) throws Exception {
     return GcsResourceId.fromGcsPath(GcsPath.fromUri(str));
   }
diff --git a/sdks/java/io/hadoop-file-system/pom.xml b/sdks/java/io/hadoop-file-system/pom.xml
index 423237b7f7fd..db5a1db1783e 100644
--- a/sdks/java/io/hadoop-file-system/pom.xml
+++ b/sdks/java/io/hadoop-file-system/pom.xml
@@ -156,6 +156,19 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-core</artifactId>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava-testlib</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest-all</artifactId>
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
new file mode 100644
index 000000000000..b0d821ba194a
--- /dev/null
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hdfs;
+
+import java.net.URI;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResourceIdTester;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Tests for {@link HadoopResourceId}.
+ */
+public class HadoopResourceIdTest {
+  private Configuration configuration;
+  private MiniDFSCluster hdfsCluster;
+  private URI hdfsClusterBaseUri;
+  private HadoopFileSystem fileSystem;
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+  @Before
+  public void setUp() throws Exception {
+    configuration = new Configuration();
+    configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.getRoot().getAbsolutePath());
+    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(configuration);
+    hdfsCluster = builder.build();
+    hdfsClusterBaseUri = new URI(configuration.get("fs.defaultFS") + "/");
+    fileSystem = new HadoopFileSystem(configuration);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    hdfsCluster.shutdown();
+  }
+
+  @Test
+  public void testResourceIdTester() throws Exception {
+    FileSystems.setDefaultConfigInWorkers(TestPipeline.testingPipelineOptions());
+    ResourceIdTester.runResourceIdBattery(new HadoopResourceId(hdfsClusterBaseUri));
+  }
+}

From ec956c85efa16d00c5e218ee1257b8ee62a2013d Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 12 May 2017 11:38:54 -0700
Subject: [PATCH 228/387] Mark FileSystem and related as Experimental

---
 .../beam/sdk/annotations/Experimental.java    |  7 +++++++
 .../java/org/apache/beam/sdk/io/AvroIO.java   |  4 ++++
 .../org/apache/beam/sdk/io/FileBasedSink.java | 21 ++++++++++++++++++-
 .../org/apache/beam/sdk/io/FileSystem.java    |  3 +++
 .../beam/sdk/io/FileSystemRegistrar.java      |  3 +++
 .../org/apache/beam/sdk/io/FileSystems.java   |  3 +++
 .../beam/sdk/io/LocalFileSystemRegistrar.java |  3 +++
 .../apache/beam/sdk/io/LocalResources.java    |  3 +++
 .../org/apache/beam/sdk/io/TFRecordIO.java    |  4 ++++
 .../java/org/apache/beam/sdk/io/TextIO.java   |  4 ++++
 .../org/apache/beam/sdk/io/fs/ResourceId.java |  3 +++
 .../gcp/storage/GcsFileSystemRegistrar.java   |  3 +++
 .../sdk/io/hdfs/HadoopFileSystemOptions.java  |  3 +++
 .../io/hdfs/HadoopFileSystemRegistrar.java    |  3 +++
 14 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
index ac02465b2f55..8224ebbe75d3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/annotations/Experimental.java
@@ -23,6 +23,7 @@
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
+
 /**
  * Signifies that a public API (public class, method or field) is subject to incompatible changes,
  * or even removal, in a future release.
@@ -79,6 +80,12 @@ enum Kind {
     /** Metrics-related experimental APIs. */
     METRICS,
 
+    /**
+     * {@link org.apache.beam.sdk.io.FileSystem} and {@link org.apache.beam.sdk.io.fs.ResourceId}
+     * related APIs.
+     */
+    FILESYSTEM,
+
     /** Experimental feature related to alternative, unnested encodings for coders. */
     CODER_CONTEXT,
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index d13c6ffb81c4..6af0e7928174 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -30,6 +30,8 @@
 import org.apache.avro.file.CodecFactory;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.reflect.ReflectData;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VoidCoder;
@@ -313,6 +315,7 @@ public Write<T> to(String outputPrefix) {
      * a common suffix (if supplied using {@link #withSuffix(String)}). This default can be
      * overridden using {@link #withFilenamePolicy(FilenamePolicy)}.
      */
+    @Experimental(Kind.FILESYSTEM)
     public Write<T> to(ResourceId outputPrefix) {
       return toResource(StaticValueProvider.of(outputPrefix));
     }
@@ -333,6 +336,7 @@ public ResourceId apply(String input) {
     /**
      * Like {@link #to(ResourceId)}.
      */
+    @Experimental(Kind.FILESYSTEM)
     public Write<T> toResource(ValueProvider<ResourceId> outputPrefix) {
       return toBuilder().setFilenamePrefix(outputPrefix).build();
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 7f729a7ea828..8102316b03f5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -47,6 +47,8 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.GZIPOutputStream;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.NullableCoder;
@@ -115,6 +117,7 @@
  *
  * @param <T> the type of values written to the sink.
  */
+@Experimental(Kind.FILESYSTEM)
 public abstract class FileBasedSink<T> implements Serializable, HasDisplayData {
   private static final Logger LOG = LoggerFactory.getLogger(FileBasedSink.class);
 
@@ -193,6 +196,7 @@ public String getFilenameSuffix() {
    * {@code /}, {@code gs://my-bucket}, or {@code c://}. In that case, interpreting the string as a
    * file will fail and this function will return a directory {@link ResourceId} instead.
    */
+  @Experimental(Kind.FILESYSTEM)
   public static ResourceId convertToFileResourceIfPossible(String outputPrefix) {
     try {
       return FileSystems.matchNewResource(outputPrefix, false /* isDirectory */);
@@ -290,6 +294,7 @@ public int getNumShards() {
      * as well as sharding information. The policy must return unique and consistent filenames
      * for different windows and panes.
      */
+    @Experimental(Kind.FILESYSTEM)
     public abstract ResourceId windowedFilename(
         ResourceId outputDirectory, WindowedContext c, String extension);
 
@@ -302,6 +307,7 @@ public abstract ResourceId windowedFilename(
      * <p>The {@link Context} object only provides sharding information, which is used by the policy
      * to generate unique and consistent filenames.
      */
+    @Experimental(Kind.FILESYSTEM)
     @Nullable public abstract ResourceId unwindowedFilename(
         ResourceId outputDirectory, Context c, String extension);
 
@@ -320,6 +326,7 @@ public void populateDisplayData(DisplayData.Builder builder) {
   /**
    * Construct a {@link FileBasedSink} with the given filename policy, producing uncompressed files.
    */
+  @Experimental(Kind.FILESYSTEM)
   public FileBasedSink(
       ValueProvider<ResourceId> baseOutputDirectoryProvider, FilenamePolicy filenamePolicy) {
     this(baseOutputDirectoryProvider, filenamePolicy, CompressionType.UNCOMPRESSED);
@@ -335,6 +342,7 @@ public ResourceId apply(ResourceId input) {
   /**
    * Construct a {@link FileBasedSink} with the given filename policy and output channel type.
    */
+  @Experimental(Kind.FILESYSTEM)
   public FileBasedSink(
       ValueProvider<ResourceId> baseOutputDirectoryProvider,
       FilenamePolicy filenamePolicy,
@@ -349,6 +357,7 @@ public FileBasedSink(
    * Returns the base directory inside which files will be written according to the configured
    * {@link FilenamePolicy}.
    */
+  @Experimental(Kind.FILESYSTEM)
   public ValueProvider<ResourceId> getBaseOutputDirectoryProvider() {
     return baseOutputDirectoryProvider;
   }
@@ -358,6 +367,7 @@ public ValueProvider<ResourceId> getBaseOutputDirectoryProvider() {
    * the {@link FilenamePolicy} may itself specify one or more inner directories before each output
    * file, say when writing windowed outputs in a {@code output/YYYY/MM/DD/file.txt} format.
    */
+  @Experimental(Kind.FILESYSTEM)
   public final FilenamePolicy getFilenamePolicy() {
     return filenamePolicy;
   }
@@ -424,9 +434,11 @@ public abstract static class WriteOperation<T> implements Serializable {
     protected final ValueProvider<ResourceId> tempDirectory;
 
     /** Whether windowed writes are being used. */
-    protected  boolean windowedWrites;
+    @Experimental(Kind.FILESYSTEM)
+    protected boolean windowedWrites;
 
     /** Constructs a temporary file resource given the temporary directory and a filename. */
+    @Experimental(Kind.FILESYSTEM)
     protected static ResourceId buildTemporaryFilename(ResourceId tempDirectory, String filename)
         throws IOException {
       return tempDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
@@ -472,6 +484,7 @@ public ResourceId apply(ResourceId baseOutputDirectory) {
      * @param sink the FileBasedSink that will be used to configure this write operation.
      * @param tempDirectory the base directory to be used for temporary output files.
      */
+    @Experimental(Kind.FILESYSTEM)
     public WriteOperation(FileBasedSink<T> sink, ResourceId tempDirectory) {
       this(sink, StaticValueProvider.of(tempDirectory));
     }
@@ -527,6 +540,7 @@ public void finalize(Iterable<FileResult> writerResults) throws Exception {
       removeTemporaryFiles(outputFilenames.keySet(), !windowedWrites);
     }
 
+    @Experimental(Kind.FILESYSTEM)
     protected final Map<ResourceId, ResourceId> buildOutputFilenames(
         Iterable<FileResult> writerResults) {
       int numShards = Iterables.size(writerResults);
@@ -610,6 +624,7 @@ public int compare(FileResult first, FileResult second) {
      * @param filenames the filenames of temporary files.
      */
     @VisibleForTesting
+    @Experimental(Kind.FILESYSTEM)
     final void copyToOutputFiles(Map<ResourceId, ResourceId> filenames)
         throws IOException {
       int numFiles = filenames.size();
@@ -637,6 +652,7 @@ final void copyToOutputFiles(Map<ResourceId, ResourceId> filenames)
      * temporary files, this method will remove them.
      */
     @VisibleForTesting
+    @Experimental(Kind.FILESYSTEM)
     final void removeTemporaryFiles(
         Set<ResourceId> knownFiles, boolean shouldRemoveTemporaryDirectory) throws IOException {
       ResourceId tempDir = tempDirectory.get();
@@ -945,6 +961,7 @@ public static final class FileResult {
     private final BoundedWindow window;
     private final PaneInfo paneInfo;
 
+    @Experimental(Kind.FILESYSTEM)
     public FileResult(ResourceId tempFilename, int shard, BoundedWindow window, PaneInfo paneInfo) {
       this.tempFilename = tempFilename;
       this.shard = shard;
@@ -952,6 +969,7 @@ public FileResult(ResourceId tempFilename, int shard, BoundedWindow window, Pane
       this.paneInfo = paneInfo;
     }
 
+    @Experimental(Kind.FILESYSTEM)
     public ResourceId getTempFilename() {
       return tempFilename;
     }
@@ -972,6 +990,7 @@ public PaneInfo getPaneInfo() {
       return paneInfo;
     }
 
+    @Experimental(Kind.FILESYSTEM)
     public ResourceId getDestinationFile(FilenamePolicy policy, ResourceId outputDirectory,
                                          int numShards, String extension) {
       checkArgument(getShard() != UNKNOWN_SHARDNUM);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystem.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystem.java
index 375264a547d0..601fecac631b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystem.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystem.java
@@ -23,6 +23,8 @@
 import java.nio.channels.WritableByteChannel;
 import java.util.Collection;
 import java.util.List;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.io.fs.CreateOptions;
 import org.apache.beam.sdk.io.fs.MatchResult;
 import org.apache.beam.sdk.io.fs.ResourceId;
@@ -35,6 +37,7 @@
  * <p>All methods are protected, and they are for file system providers to implement.
  * Clients should use {@link FileSystems} utility.
  */
+@Experimental(Kind.FILESYSTEM)
 public abstract class FileSystem<ResourceIdT extends ResourceId> {
   /**
    * This is the entry point to convert user-provided specs to {@link ResourceIdT ResourceIds}.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemRegistrar.java
index 78a91f6b6ab7..50ee6ebda1e5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemRegistrar.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemRegistrar.java
@@ -20,6 +20,8 @@
 import com.google.auto.service.AutoService;
 import java.util.ServiceLoader;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.options.PipelineOptions;
 
 /**
@@ -31,6 +33,7 @@
  * <p>It is optional but recommended to use one of the many build time tools such as
  * {@link AutoService} to generate the necessary META-INF files automatically.
  */
+@Experimental(Kind.FILESYSTEM)
 public interface FileSystemRegistrar {
   /**
    * Create zero or more {@link FileSystem filesystems} from the given {@link PipelineOptions}.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index 4341fab65450..cfb63c086de4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -49,6 +49,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import javax.annotation.Nonnull;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.io.fs.CreateOptions;
 import org.apache.beam.sdk.io.fs.CreateOptions.StandardCreateOptions;
 import org.apache.beam.sdk.io.fs.MatchResult;
@@ -63,6 +65,7 @@
 /**
  * Clients facing {@link FileSystem} utility.
  */
+@Experimental(Kind.FILESYSTEM)
 public class FileSystems {
 
   public static final String DEFAULT_SCHEME = "default";
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java
index f18236056c87..7896e203ed30 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java
@@ -20,12 +20,15 @@
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.options.PipelineOptions;
 
 /**
  * {@link AutoService} registrar for the {@link LocalFileSystem}.
  */
 @AutoService(FileSystemRegistrar.class)
+@Experimental(Kind.FILESYSTEM)
 public class LocalFileSystemRegistrar implements FileSystemRegistrar {
   @Override
   public Iterable<FileSystem> fromOptions(@Nullable PipelineOptions options) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResources.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResources.java
index 817829b1056d..d234bae8371b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResources.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResources.java
@@ -20,6 +20,8 @@
 import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
@@ -28,6 +30,7 @@
 /**
  * Helper functions for producing a {@link ResourceId} that references a local file or directory.
  */
+@Experimental(Kind.FILESYSTEM)
 public final class LocalResources {
 
   public static ResourceId fromFile(File file, boolean isDirectory) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
index f73d6f3d284f..c2745957bee2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
@@ -31,6 +31,8 @@
 import java.nio.channels.WritableByteChannel;
 import java.util.NoSuchElementException;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VoidCoder;
@@ -277,6 +279,7 @@ public Write to(String outputPrefix) {
      *
      * <p>For more information on filenames, see {@link DefaultFilenamePolicy}.
      */
+    @Experimental(Kind.FILESYSTEM)
     public Write to(ResourceId outputResource) {
       return toResource(StaticValueProvider.of(outputResource));
     }
@@ -284,6 +287,7 @@ public Write to(ResourceId outputResource) {
     /**
      * Like {@link #to(ResourceId)}.
      */
+    @Experimental(Kind.FILESYSTEM)
     public Write toResource(ValueProvider<ResourceId> outputResource) {
       return toBuilder().setOutputPrefix(outputResource).build();
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
index af6a06967fc7..5c068ce06441 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
@@ -23,6 +23,8 @@
 
 import com.google.auto.value.AutoValue;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VoidCoder;
@@ -306,6 +308,7 @@ public Write to(String filenamePrefix) {
      * in which case {@link #withShardNameTemplate(String)} and {@link #withSuffix(String)} should
      * not be set.
      */
+    @Experimental(Kind.FILESYSTEM)
     public Write to(ResourceId filenamePrefix) {
       return toResource(StaticValueProvider.of(filenamePrefix));
     }
@@ -326,6 +329,7 @@ public ResourceId apply(String input) {
     /**
      * Like {@link #to(ResourceId)}.
      */
+    @Experimental(Kind.FILESYSTEM)
     public Write toResource(ValueProvider<ResourceId> filenamePrefix) {
       return toBuilder().setFilenamePrefix(filenamePrefix).build();
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
index dfe771f371ba..9196034321b7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
@@ -19,6 +19,8 @@
 
 import java.io.Serializable;
 import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.io.FileSystem;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
@@ -45,6 +47,7 @@
  *   to generate {@link ResourceId ResourceIds} for resources that may not yet exist.
  * </ul>
  */
+@Experimental(Kind.FILESYSTEM)
 public interface ResourceId extends Serializable {
 
   /**
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java
index 9f5980a700ad..f954b33de024 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java
@@ -22,6 +22,8 @@
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import javax.annotation.Nonnull;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.io.FileSystem;
 import org.apache.beam.sdk.io.FileSystemRegistrar;
@@ -31,6 +33,7 @@
  * {@link AutoService} registrar for the {@link GcsFileSystem}.
  */
 @AutoService(FileSystemRegistrar.class)
+@Experimental(Kind.FILESYSTEM)
 public class GcsFileSystemRegistrar implements FileSystemRegistrar {
 
   @Override
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java
index 45f43e23c80d..f5183d516168 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java
@@ -25,6 +25,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
@@ -38,6 +40,7 @@
  * {@link PipelineOptions} which encapsulate {@link Configuration Hadoop Configuration}
  * for the {@link HadoopFileSystem}.
  */
+@Experimental(Kind.FILESYSTEM)
 public interface HadoopFileSystemOptions extends PipelineOptions {
   @Description("A list of Hadoop configurations used to configure zero or more Hadoop filesystems. "
       + "By default, Hadoop configuration is loaded from 'core-site.xml' and 'hdfs-site.xml' "
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java
index 9159df3a11f1..8c570892deaa 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java
@@ -25,6 +25,8 @@
 import java.util.Collections;
 import java.util.List;
 import javax.annotation.Nonnull;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.io.FileSystem;
 import org.apache.beam.sdk.io.FileSystemRegistrar;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -34,6 +36,7 @@
  * {@link AutoService} registrar for the {@link HadoopFileSystem}.
  */
 @AutoService(FileSystemRegistrar.class)
+@Experimental(Kind.FILESYSTEM)
 public class HadoopFileSystemRegistrar implements FileSystemRegistrar {
 
   @Override

From 15df211c758e7c8f05c3136f25bbe18e3f394321 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 12 May 2017 11:11:06 -0700
Subject: [PATCH 229/387] [BEAM-2277] HadoopFileSystem: normalize
 implementation

* Drop empty authority always
* Resolve directories
---
 .../beam/sdk/io/fs/ResourceIdTester.java      | 15 ++++-----
 .../beam/sdk/io/hdfs/HadoopFileSystem.java    | 32 +++++++++++--------
 .../beam/sdk/io/hdfs/HadoopResourceId.java    | 16 +++++++++-
 .../sdk/io/hdfs/HadoopFileSystemTest.java     |  3 +-
 .../sdk/io/hdfs/HadoopResourceIdTest.java     | 22 +++++++++----
 5 files changed, 56 insertions(+), 32 deletions(-)

diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java
index fe50adac6983..8ceaeed5359a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java
@@ -21,9 +21,8 @@
 import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY;
 import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE;
 import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertFalse;
+import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import com.google.common.testing.EqualsTester;
@@ -66,16 +65,16 @@ private static void validateResolvingIds(
     ResourceId file2a = baseDirectory.resolve("child2", RESOLVE_FILE);
     allResourceIds.add(file1);
     allResourceIds.add(file2);
-    assertFalse("Resolved file isDirectory()", file1.isDirectory());
-    assertFalse("Resolved file isDirectory()", file2.isDirectory());
-    assertFalse("Resolved file isDirectory()", file2a.isDirectory());
+    assertThat("Resolved file isDirectory()", file1.isDirectory(), is(false));
+    assertThat("Resolved file isDirectory()", file2.isDirectory(), is(false));
+    assertThat("Resolved file isDirectory()", file2a.isDirectory(), is(false));
 
     ResourceId dir1 = baseDirectory.resolve("child1", RESOLVE_DIRECTORY);
     ResourceId dir2 = baseDirectory.resolve("child2", RESOLVE_DIRECTORY);
     ResourceId dir2a = baseDirectory.resolve("child2", RESOLVE_DIRECTORY);
-    assertTrue("Resolved directory isDirectory()", dir1.isDirectory());
-    assertTrue("Resolved directory isDirectory()", dir2.isDirectory());
-    assertTrue("Resolved directory isDirectory()", dir2a.isDirectory());
+    assertThat("Resolved directory isDirectory()", dir1.isDirectory(), is(true));
+    assertThat("Resolved directory isDirectory()", dir2.isDirectory(), is(true));
+    assertThat("Resolved directory isDirectory()", dir2a.isDirectory(), is(true));
     allResourceIds.add(dir1);
     allResourceIds.add(dir2);
 
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
index 154a818e99b8..d519a8c86266 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
@@ -21,7 +21,6 @@
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
@@ -82,8 +81,9 @@ protected List<MatchResult> match(List<String> specs) {
         List<Metadata> metadata = new ArrayList<>();
         for (FileStatus fileStatus : fileStatuses) {
           if (fileStatus.isFile()) {
+            URI uri = dropEmptyAuthority(fileStatus.getPath().toUri().toString());
             metadata.add(Metadata.builder()
-                .setResourceId(new HadoopResourceId(fileStatus.getPath().toUri()))
+                .setResourceId(new HadoopResourceId(uri))
                 .setIsReadSeekEfficient(true)
                 .setSizeBytes(fileStatus.getLen())
                 .build());
@@ -151,19 +151,13 @@ protected void delete(Collection<HadoopResourceId> resourceIds) throws IOExcepti
 
   @Override
   protected HadoopResourceId matchNewResource(String singleResourceSpec, boolean isDirectory) {
-    try {
-      if (singleResourceSpec.endsWith("/") && !isDirectory) {
-        throw new IllegalArgumentException(String.format(
-            "Expected file path but received directory path %s", singleResourceSpec));
-      }
-      return !singleResourceSpec.endsWith("/") && isDirectory
-          ? new HadoopResourceId(new URI(singleResourceSpec + "/"))
-          : new HadoopResourceId(new URI(singleResourceSpec));
-    } catch (URISyntaxException e) {
-      throw new IllegalArgumentException(
-          String.format("Invalid spec %s directory %s", singleResourceSpec, isDirectory),
-          e);
+    if (singleResourceSpec.endsWith("/") && !isDirectory) {
+      throw new IllegalArgumentException(String.format(
+          "Expected file path but received directory path %s", singleResourceSpec));
     }
+    return !singleResourceSpec.endsWith("/") && isDirectory
+        ? new HadoopResourceId(dropEmptyAuthority(singleResourceSpec + "/"))
+        : new HadoopResourceId(dropEmptyAuthority(singleResourceSpec));
   }
 
   @Override
@@ -237,4 +231,14 @@ public void close() throws IOException {
       inputStream.close();
     }
   }
+
+  private static URI dropEmptyAuthority(String uriStr) {
+    URI uri = URI.create(uriStr);
+    String prefix = uri.getScheme() + ":///";
+    if (uriStr.startsWith(prefix)) {
+      return URI.create(uri.getScheme() + ":/" + uriStr.substring(prefix.length()));
+    } else {
+      return uri;
+    }
+  }
 }
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
index e5708649589c..88fa32a3a08c 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
@@ -17,9 +17,12 @@
  */
 package org.apache.beam.sdk.io.hdfs;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import java.net.URI;
 import java.util.Objects;
 import org.apache.beam.sdk.io.fs.ResolveOptions;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.hadoop.fs.Path;
 
@@ -35,7 +38,18 @@ class HadoopResourceId implements ResourceId {
 
   @Override
   public ResourceId resolve(String other, ResolveOptions resolveOptions) {
-    return new HadoopResourceId(uri.resolve(other));
+    if (resolveOptions == StandardResolveOptions.RESOLVE_DIRECTORY) {
+      if (!other.endsWith("/")) {
+        other += '/';
+      }
+      return new HadoopResourceId(uri.resolve(other));
+    } else if (resolveOptions == StandardResolveOptions.RESOLVE_FILE) {
+      checkArgument(!other.endsWith("/"), "Resolving a file with a directory path: %s", other);
+      return new HadoopResourceId(uri.resolve(other));
+    } else {
+      throw new UnsupportedOperationException(
+          String.format("Unexpected StandardResolveOptions %s", resolveOptions));
+    }
   }
 
   @Override
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
index cf86c36b52c2..14591d8b664c 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
@@ -63,14 +63,13 @@ public class HadoopFileSystemTest {
   @Rule public TestPipeline p = TestPipeline.create();
   @Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
   @Rule public ExpectedException thrown = ExpectedException.none();
-  private Configuration configuration;
   private MiniDFSCluster hdfsCluster;
   private URI hdfsClusterBaseUri;
   private HadoopFileSystem fileSystem;
 
   @Before
   public void setUp() throws Exception {
-    configuration = new Configuration();
+    Configuration configuration = new Configuration();
     configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.getRoot().getAbsolutePath());
     MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(configuration);
     hdfsCluster = builder.build();
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
index b0d821ba194a..f17913270d5d 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
@@ -18,9 +18,11 @@
 package org.apache.beam.sdk.io.hdfs;
 
 import java.net.URI;
+import java.util.Collections;
 import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.fs.ResourceIdTester;
-import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.After;
@@ -33,21 +35,25 @@
  * Tests for {@link HadoopResourceId}.
  */
 public class HadoopResourceIdTest {
-  private Configuration configuration;
+
   private MiniDFSCluster hdfsCluster;
   private URI hdfsClusterBaseUri;
-  private HadoopFileSystem fileSystem;
+
   @Rule
   public TemporaryFolder tmpFolder = new TemporaryFolder();
 
   @Before
   public void setUp() throws Exception {
-    configuration = new Configuration();
+    Configuration configuration = new Configuration();
     configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.getRoot().getAbsolutePath());
     MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(configuration);
     hdfsCluster = builder.build();
     hdfsClusterBaseUri = new URI(configuration.get("fs.defaultFS") + "/");
-    fileSystem = new HadoopFileSystem(configuration);
+
+    // Register HadoopFileSystem for this test.
+    HadoopFileSystemOptions options = PipelineOptionsFactory.as(HadoopFileSystemOptions.class);
+    options.setHdfsConfiguration(Collections.singletonList(configuration));
+    FileSystems.setDefaultConfigInWorkers(options);
   }
 
   @After
@@ -57,7 +63,9 @@ public void tearDown() throws Exception {
 
   @Test
   public void testResourceIdTester() throws Exception {
-    FileSystems.setDefaultConfigInWorkers(TestPipeline.testingPipelineOptions());
-    ResourceIdTester.runResourceIdBattery(new HadoopResourceId(hdfsClusterBaseUri));
+    ResourceId baseDirectory =
+        FileSystems.matchNewResource(
+            "hdfs://" + hdfsClusterBaseUri.getPath(), true /* isDirectory */);
+    ResourceIdTester.runResourceIdBattery(baseDirectory);
   }
 }

From f3540d47f10c18859340a738a7e93643ee57f604 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 12 May 2017 11:46:16 -0700
Subject: [PATCH 230/387] Rename FileSystems.setDefaultConfigInWorkers

And document that it's not for users.
---
 .../utils/SerializablePipelineOptions.java           |  2 +-
 .../translation/utils/SerializedPipelineOptions.java |  2 +-
 .../dataflow/DataflowPipelineTranslatorTest.java     |  2 +-
 .../beam/runners/dataflow/DataflowRunnerTest.java    |  6 +++---
 .../options/DataflowPipelineOptionsTest.java         |  4 ++--
 .../beam/runners/dataflow/util/PackageUtilTest.java  |  2 +-
 .../spark/translation/SparkRuntimeContext.java       |  2 +-
 .../java/org/apache/beam/sdk/PipelineRunner.java     |  2 +-
 .../java/org/apache/beam/sdk/io/FileSystems.java     | 12 +++++++++++-
 .../org/apache/beam/sdk/testing/TestPipeline.java    |  2 +-
 .../gcp/storage/GcsFileSystemRegistrar.java          |  2 +-
 .../extensions/gcp/storage/GcsResourceIdTest.java    |  2 +-
 .../beam/sdk/io/hdfs/HadoopFileSystemTest.java       |  2 +-
 .../beam/sdk/io/hdfs/HadoopResourceIdTest.java       |  2 +-
 14 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
index 02afa7a31e46..46b04fc712fb 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/SerializablePipelineOptions.java
@@ -62,7 +62,7 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept
         .as(ApexPipelineOptions.class);
 
     if (FILE_SYSTEMS_INTIIALIZED.compareAndSet(false, true)) {
-      FileSystems.setDefaultConfigInWorkers(pipelineOptions);
+      FileSystems.setDefaultPipelineOptions(pipelineOptions);
     }
   }
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
index 84f3bf40f59c..40b6dd615562 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerializedPipelineOptions.java
@@ -56,7 +56,7 @@ public PipelineOptions getPipelineOptions() {
       try {
         pipelineOptions = createMapper().readValue(serializedOptions, PipelineOptions.class);
 
-        FileSystems.setDefaultConfigInWorkers(pipelineOptions);
+        FileSystems.setDefaultPipelineOptions(pipelineOptions);
       } catch (IOException e) {
         throw new RuntimeException("Couldn't deserialize the PipelineOptions.", e);
       }
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 93c1e5b109cc..87744f0f1436 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -144,7 +144,7 @@ private Pipeline buildPipeline(DataflowPipelineOptions options) {
     Pipeline p = Pipeline.create(options);
 
     // Enable the FileSystems API to know about gs:// URIs in this test.
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
 
     p.apply("ReadMyFile", TextIO.read().from("gs://bucket/object"))
      .apply("WriteMyFile", TextIO.write().to("gs://bucket/object"));
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index ce01aa11ba2b..8f10b18d3eda 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -177,7 +177,7 @@ private Pipeline buildDataflowPipeline(DataflowPipelineOptions options) {
         .apply("WriteMyFile", TextIO.write().to("gs://bucket/object"));
 
     // Enable the FileSystems API to know about gs:// URIs in this test.
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
 
     return p;
   }
@@ -246,7 +246,7 @@ private DataflowPipelineOptions buildPipelineOptions() throws IOException {
     options.setGcpCredential(new TestCredential());
 
     // Configure the FileSystem registrar to use these options.
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
 
     return options;
   }
@@ -771,7 +771,7 @@ public void testProjectDescription() throws IOException {
   @Test
   public void testInvalidNumberOfWorkerHarnessThreads() throws IOException {
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
     options.setRunner(DataflowRunner.class);
     options.setProject("foo-12345");
 
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
index 613604ac8426..cea44f083dc6 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
@@ -137,7 +137,7 @@ public void testStagingLocation() {
   @Test
   public void testDefaultToTempLocation() {
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
     options.setPathValidatorClass(NoopPathValidator.class);
     options.setTempLocation("gs://temp_location/");
     assertEquals("gs://temp_location/", options.getGcpTempLocation());
@@ -147,7 +147,7 @@ public void testDefaultToTempLocation() {
   @Test
   public void testDefaultToGcpTempLocation() {
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
     options.setPathValidatorClass(NoopPathValidator.class);
     options.setTempLocation("gs://temp_location/");
     options.setGcpTempLocation("gs://gcp_temp_location/");
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
index c7a660e688fb..5d0c0f2d68a6 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
@@ -120,7 +120,7 @@ public void setUp() {
 
     GcsOptions pipelineOptions = PipelineOptionsFactory.as(GcsOptions.class);
     pipelineOptions.setGcsUtil(mockGcsUtil);
-    FileSystems.setDefaultConfigInWorkers(pipelineOptions);
+    FileSystems.setDefaultPipelineOptions(pipelineOptions);
     createOptions = StandardCreateOptions.builder().setMimeType(MimeTypes.BINARY).build();
   }
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index 82e8b440ab8d..f3fe99c9c074 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -90,7 +90,7 @@ static PipelineOptions getOrInit(String serializedPipelineOptions) {
           }
         }
         // Register standard FileSystems.
-        FileSystems.setDefaultConfigInWorkers(pipelineOptions);
+        FileSystems.setDefaultPipelineOptions(pipelineOptions);
       }
       return pipelineOptions;
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
index 87705af22d6a..c114501724a2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java
@@ -41,7 +41,7 @@ public static PipelineRunner<? extends PipelineResult> fromOptions(PipelineOptio
     PipelineOptionsValidator.validate(PipelineOptions.class, options);
 
     // (Re-)register standard FileSystems. Clobbers any prior credentials.
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
 
     @SuppressWarnings("unchecked")
     PipelineRunner<? extends PipelineResult> result =
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index cfb63c086de4..1aacc9020705 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -51,6 +51,7 @@
 import javax.annotation.Nonnull;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.io.fs.CreateOptions;
 import org.apache.beam.sdk.io.fs.CreateOptions.StandardCreateOptions;
 import org.apache.beam.sdk.io.fs.MatchResult;
@@ -448,12 +449,21 @@ static FileSystem getFileSystemInternal(String scheme) {
 
   /********************************** METHODS FOR REGISTRATION **********************************/
 
+  /** @deprecated to be removed. */
+  @Deprecated // for DataflowRunner backwards compatibility.
+  public static void setDefaultConfigInWorkers(PipelineOptions options) {
+    setDefaultPipelineOptions(options);
+  }
+
   /**
    * Sets the default configuration in workers.
    *
    * <p>It will be used in {@link FileSystemRegistrar FileSystemRegistrars} for all schemes.
+   *
+   * <p>This is expected only to be used by runners after {@code Pipeline.run}, or in tests.
    */
-  public static void setDefaultConfigInWorkers(PipelineOptions options) {
+  @Internal
+  public static void setDefaultPipelineOptions(PipelineOptions options) {
     checkNotNull(options, "options");
     Set<FileSystemRegistrar> registrars =
         Sets.newTreeSet(ReflectHelpers.ObjectsClassComparator.INSTANCE);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
index e04c2f835ab5..9206e0444819 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
@@ -413,7 +413,7 @@ public static PipelineOptions testingPipelineOptions() {
       }
       options.setStableUniqueNames(CheckEnabled.ERROR);
 
-      FileSystems.setDefaultConfigInWorkers(options);
+      FileSystems.setDefaultPipelineOptions(options);
       return options;
     } catch (IOException e) {
       throw new RuntimeException(
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java
index f954b33de024..f3a67b2f83df 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java
@@ -40,7 +40,7 @@ public class GcsFileSystemRegistrar implements FileSystemRegistrar {
   public Iterable<FileSystem> fromOptions(@Nonnull PipelineOptions options) {
     checkNotNull(
         options,
-        "Expect the runner have called FileSystems.setDefaultConfigInWorkers().");
+        "Expect the runner have called FileSystems.setDefaultPipelineOptions().");
     return ImmutableList.<FileSystem>of(new GcsFileSystem(options.as(GcsOptions.class)));
   }
 }
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
index 2a6750185f12..c1e214e5bb45 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
@@ -168,7 +168,7 @@ public void testGetFilename() throws Exception {
 
   @Test
   public void testResourceIdTester() throws Exception {
-    FileSystems.setDefaultConfigInWorkers(TestPipeline.testingPipelineOptions());
+    FileSystems.setDefaultPipelineOptions(TestPipeline.testingPipelineOptions());
     ResourceIdTester.runResourceIdBattery(toResourceIdentifier("gs://bucket/foo/"));
   }
 
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
index 14591d8b664c..88275f49b05c 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
@@ -219,7 +219,7 @@ public void testReadPipeline() throws Exception {
     HadoopFileSystemOptions options = TestPipeline.testingPipelineOptions()
         .as(HadoopFileSystemOptions.class);
     options.setHdfsConfiguration(ImmutableList.of(fileSystem.fileSystem.getConf()));
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
     PCollection<String> pc = p.apply(
         TextIO.read().from(testPath("testFile*").toString()));
     PAssert.that(pc).containsInAnyOrder("testDataA", "testDataB", "testDataC");
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
index f17913270d5d..c4a8577fbd87 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopResourceIdTest.java
@@ -53,7 +53,7 @@ public void setUp() throws Exception {
     // Register HadoopFileSystem for this test.
     HadoopFileSystemOptions options = PipelineOptionsFactory.as(HadoopFileSystemOptions.class);
     options.setHdfsConfiguration(Collections.singletonList(configuration));
-    FileSystems.setDefaultConfigInWorkers(options);
+    FileSystems.setDefaultPipelineOptions(options);
   }
 
   @After

From 3921163829d2a99b0524bf3fa1b85a4cde826f3a Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 12 May 2017 13:57:59 -0700
Subject: [PATCH 231/387] Fix shading of guava testlib

---
 pom.xml                     | 4 ++++
 runners/direct-java/pom.xml | 4 ++++
 sdks/java/core/pom.xml      | 4 ++++
 3 files changed, 12 insertions(+)

diff --git a/pom.xml b/pom.xml
index eaca6b7ff661..a978f58fe3b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1446,6 +1446,10 @@
                 <relocations>
                   <relocation>
                     <pattern>com.google.common</pattern>
+                    <excludes>
+                      <!-- com.google.common is too generic, need to exclude guava-testlib -->
+                      <exclude>com.google.common.**.testing.*</exclude>
+                    </excludes>
                     <!--suppress MavenModelInspection -->
                     <shadedPattern>
                       org.apache.${renderedArtifactId}.repackaged.com.google.common
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 0daa066221a3..c581113120dd 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -131,6 +131,10 @@
                 </relocation>
                 <relocation>
                   <pattern>com.google.common</pattern>
+                  <excludes>
+                    <!-- com.google.common is too generic, need to exclude guava-testlib -->
+                    <exclude>com.google.common.**.testing.*</exclude>
+                  </excludes>
                   <shadedPattern>
                     org.apache.beam.runners.direct.repackaged.com.google.common
                   </shadedPattern>
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 68d76e17021e..882657b1db4e 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -161,6 +161,10 @@
               <relocations>
                 <relocation>
                   <pattern>com.google.common</pattern>
+                  <excludes>
+                    <!-- com.google.common is too generic, need to exclude guava-testlib -->
+                    <exclude>com.google.common.**.testing.*</exclude>
+                  </excludes>
                   <!--suppress MavenModelInspection -->
                   <shadedPattern>
                     org.apache.beam.sdk.repackaged.com.google.common

From f53ceecfb18a52aeeab8f1869c189e4aa2f56d35 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Fri, 12 May 2017 14:54:33 -0700
Subject: [PATCH 232/387] internal comments

---
 sdks/python/apache_beam/internal/pickler.py   | 2 ++
 sdks/python/apache_beam/internal/util.py      | 5 ++++-
 sdks/python/apache_beam/transforms/trigger.py | 6 +++++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index 4305379747b3..e049a7134231 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -17,6 +17,8 @@
 
 """Pickler for values, functions, and classes.
 
+For internal use only. No backwards compatibility guarantees.
+
 Pickles created by the pickling library contain non-ASCII characters, so
 we base64-encode the results so that we can put them in a JSON objects.
 The pickler is used to embed FlatMap callable objects into the workflow JSON
diff --git a/sdks/python/apache_beam/internal/util.py b/sdks/python/apache_beam/internal/util.py
index df4878c31382..dbbeafcc1e75 100644
--- a/sdks/python/apache_beam/internal/util.py
+++ b/sdks/python/apache_beam/internal/util.py
@@ -15,7 +15,10 @@
 # limitations under the License.
 #
 
-"""Utility functions used throughout the package."""
+"""Utility functions used throughout the package.
+
+For internal use only. No backwards compatibility guarantees.
+"""
 
 import logging
 from multiprocessing.pool import ThreadPool
diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py
index 7de2f85100a8..42009958552a 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -37,6 +37,7 @@
 from apache_beam.utils.timestamp import MAX_TIMESTAMP
 from apache_beam.utils.timestamp import MIN_TIMESTAMP
 
+# AfterCount is experimental. No backwards compatibility guarantees.
 
 __all__ = [
     'AccumulationMode',
@@ -367,7 +368,10 @@ def to_runner_api(self, context):
 
 
 class AfterCount(TriggerFn):
-  """Fire when there are at least count elements in this window pane."""
+  """Fire when there are at least count elements in this window pane.
+
+  AfterCount is experimental. No backwards compatibility guarantees.
+  """
 
   COUNT_TAG = _CombiningValueStateTag('count', combiners.CountCombineFn())
 

From 7bade36eb18a48e01a8401f97f10a70a492172c4 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 12 May 2017 15:52:51 -0700
Subject: [PATCH 233/387] [BEAM-2279] Fix archetype breakages

* Add version for hadoop-file-system dependency.
* Add DirectRunner in test scope in java8, otherwise mvn package fails
  when another runner profile is enabled.
---
 .../src/main/resources/archetype-resources/pom.xml        | 8 ++++++++
 .../src/main/resources/archetype-resources/pom.xml        | 1 +
 2 files changed, 9 insertions(+)

diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 05826baf6ccb..5f34689f028f 100644
--- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -218,6 +218,7 @@
         <dependency>
           <groupId>org.apache.beam</groupId>
           <artifactId>beam-sdks-java-io-hadoop-file-system</artifactId>
+          <version>${beam.version}</version>
           <scope>runtime</scope>
         </dependency>
         <dependency>
@@ -361,5 +362,12 @@
       <version>1.9.5</version>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-runners-direct-java</artifactId>
+      <version>${beam.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index fb6e8b18d633..a3d7b8fd73b6 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -218,6 +218,7 @@
         <dependency>
           <groupId>org.apache.beam</groupId>
           <artifactId>beam-sdks-java-io-hadoop-file-system</artifactId>
+          <version>${beam.version}</version>
           <scope>runtime</scope>
         </dependency>
         <dependency>

From 504bd6a8036946a796a7290853e45bf20d32d07d Mon Sep 17 00:00:00 2001
From: Rune Fevang <fevang@exabel.com>
Date: Fri, 12 May 2017 22:24:32 +0200
Subject: [PATCH 234/387] [BEAM-2290] Fix issue where timestamps weren't set
 when using CompressedSource

---
 .../apache/beam/sdk/io/CompressedSource.java  |  6 +++++
 .../beam/sdk/io/CompressedSourceTest.java     | 25 +++++++++++++++++--
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
index f2fc37bcef6e..6ab8dec3db32 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
@@ -42,6 +42,7 @@
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.joda.time.Instant;
 
 /**
  * A Source that reads from compressed files. A {@code CompressedSources} wraps a delegate
@@ -579,5 +580,10 @@ protected final long getCurrentOffset() throws NoSuchElementException {
         return channel.getCount();
       }
     }
+
+    @Override
+    public Instant getCurrentTimestamp() throws NoSuchElementException {
+      return readerDelegate.getCurrentTimestamp();
+    }
   }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
index 014e16e7c349..3fff31993306 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
@@ -64,12 +64,16 @@
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.SourceTestUtils;
 import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
 import org.hamcrest.Matchers;
+import org.joda.time.Instant;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -552,8 +556,13 @@ private void verifyReadContents(byte[] expected, File inputFile,
     if (decompressionFactory != null) {
       source = source.withDecompression(decompressionFactory);
     }
-    PCollection<Byte> output = p.apply(Read.from(source));
-    PAssert.that(output).containsInAnyOrder(Bytes.asList(expected));
+    PCollection<KV<Long, Byte>> output = p.apply(Read.from(source))
+        .apply(ParDo.of(new ExtractIndexFromTimestamp()));
+    ArrayList<KV<Long, Byte>> expectedOutput = new ArrayList<>();
+    for (int i = 0; i < expected.length; i++) {
+      expectedOutput.add(KV.of((long) i, expected[i]));
+    }
+    PAssert.that(output).containsInAnyOrder(expectedOutput);
     p.run();
   }
 
@@ -632,6 +641,18 @@ protected boolean readNextRecord() throws IOException {
       protected long getCurrentOffset() {
         return offset;
       }
+
+      @Override
+      public Instant getCurrentTimestamp() throws NoSuchElementException {
+        return new Instant(getCurrentOffset());
+      }
+    }
+  }
+
+  private static class ExtractIndexFromTimestamp extends DoFn<Byte, KV<Long, Byte>> {
+    @ProcessElement
+    public void processElement(ProcessContext context) {
+      context.output(KV.of(context.timestamp().getMillis(), context.element()));
     }
   }
 

From 8060d66083973cdab91f68085652ab68e3dee596 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Mon, 15 May 2017 10:00:45 -0700
Subject: [PATCH 235/387] Reduce Log Level of PubsubUnboundedSource

---
 .../apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index e8fe7011c121..b7df804f098a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -863,7 +863,7 @@ private void stats() {
             (nowMsSinceEpoch - inFlight.get(oldestAckId).requestTimeMsSinceEpoch) + "ms";
       }
 
-      LOG.info("Pubsub {} has "
+      LOG.debug("Pubsub {} has "
                + "{} received messages, "
                + "{} current unread messages, "
                + "{} current unread bytes, "

From de1adefda958ae152eae4b7b94b3b78563e130aa Mon Sep 17 00:00:00 2001
From: Neville Li <neville.lyh@gmail.com>
Date: Mon, 15 May 2017 14:55:09 -0400
Subject: [PATCH 236/387] readAvros should't have proto Message upper bound

---
 .../main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index bdc2752e7fa9..e293b95ede1e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -487,7 +487,7 @@ public static <T extends Message> Read<T> readProtos(Class<T> messageClass) {
    * Returns A {@link PTransform} that continuously reads binary encoded Avro messages of the
    * given type from a Google Cloud Pub/Sub stream.
    */
-  public static <T extends Message> Read<T> readAvros(Class<T> clazz) {
+  public static <T> Read<T> readAvros(Class<T> clazz) {
     // TODO: Stop using AvroCoder and instead parse the payload directly.
     // We should not be relying on the fact that AvroCoder's wire format is identical to
     // the Avro wire format, as the wire format is not part of a coder's API.

From 15ec2cd83d4291957a75d9f2ac838ff867dcbe0c Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Mon, 15 May 2017 10:57:24 -0700
Subject: [PATCH 237/387] Fix GcsResourceIdTest in postcommits

This test is not a @ValidatesRunner test, so it should not use TestPipelineOptions.
---
 .../beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
index c1e214e5bb45..ae7d40c81a15 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceIdTest.java
@@ -26,7 +26,7 @@
 import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.io.fs.ResourceIdTester;
-import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.junit.Rule;
 import org.junit.Test;
@@ -168,7 +168,7 @@ public void testGetFilename() throws Exception {
 
   @Test
   public void testResourceIdTester() throws Exception {
-    FileSystems.setDefaultPipelineOptions(TestPipeline.testingPipelineOptions());
+    FileSystems.setDefaultPipelineOptions(PipelineOptionsFactory.create());
     ResourceIdTester.runResourceIdBattery(toResourceIdentifier("gs://bucket/foo/"));
   }
 

From 911128b287909e2e2240e0674223df7c66e952f8 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Mon, 15 May 2017 09:59:29 -0700
Subject: [PATCH 238/387] Moving the data file for trigger tests to
 testing/data

---
 .../{transforms => testing/data}/trigger_transcripts.yaml    | 0
 sdks/python/apache_beam/transforms/trigger_test.py           | 5 +++--
 2 files changed, 3 insertions(+), 2 deletions(-)
 rename sdks/python/apache_beam/{transforms => testing/data}/trigger_transcripts.yaml (100%)

diff --git a/sdks/python/apache_beam/transforms/trigger_transcripts.yaml b/sdks/python/apache_beam/testing/data/trigger_transcripts.yaml
similarity index 100%
rename from sdks/python/apache_beam/transforms/trigger_transcripts.yaml
rename to sdks/python/apache_beam/testing/data/trigger_transcripts.yaml
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index 4a5affb6a30a..1ae1f0250320 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -590,8 +590,9 @@ def fire_timers():
     self.assertEquals([], output, msg='Unexpected output: %s' % output)
 
 
-TRANSCRIPT_TEST_FILE = os.path.join(os.path.dirname(__file__),
-                                    'trigger_transcripts.yaml')
+TRANSCRIPT_TEST_FILE = os.path.join(
+    os.path.dirname(__file__), '..', 'testing', 'data',
+    'trigger_transcripts.yaml')
 if os.path.exists(TRANSCRIPT_TEST_FILE):
   TranscriptTest._create_tests(TRANSCRIPT_TEST_FILE)
 

From 96906aee9d0cdbb9fae67dd5bd2af0710a1b67bf Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Mon, 15 May 2017 11:26:59 -0700
Subject: [PATCH 239/387] Remove "Dataflow" from apache_beam __init__.py file

---
 sdks/python/apache_beam/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py
index 992149848f94..8b772c9e875b 100644
--- a/sdks/python/apache_beam/__init__.py
+++ b/sdks/python/apache_beam/__init__.py
@@ -68,7 +68,7 @@
 
 if not (sys.version_info[0] == 2 and sys.version_info[1] == 7):
   raise RuntimeError(
-      'Dataflow SDK for Python is supported only on Python 2.7. '
+      'The Apache Beam SDK for Python is supported only on Python 2.7. '
       'It is not supported on Python ['+ str(sys.version_info) + '].')
 
 # pylint: disable=wrong-import-position

From 044d1832f44394f2dc61172d4a2737157a4e2210 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Mon, 15 May 2017 09:40:49 -0700
Subject: [PATCH 240/387] [BEAM-2299] Run maven install on Windows machine for
 build/test coverage on Windows

---
 .../jenkins/common_job_properties.groovy      | 29 ++++++++----
 ...ostCommit_Java_MavenInstall_Windows.groovy | 45 +++++++++++++++++++
 sdks/common/runner-api/pom.xml                |  3 --
 3 files changed, 65 insertions(+), 12 deletions(-)
 create mode 100644 .test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy

diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
index 56eb0dedd028..f47ab282a698 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -24,14 +24,25 @@ class common_job_properties {
 
   // Sets common top-level job properties for website repository jobs.
   static void setTopLevelWebsiteJobProperties(context) {
-    setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30)
+    setTopLevelJobProperties(
+            context,
+            'beam-site',
+            'asf-site',
+            'beam',
+            30)
   }
 
   // Sets common top-level job properties for main repository jobs.
   static void setTopLevelMainJobProperties(context,
-                                           String defaultBranch = 'master',
-                                           int defaultTimeout = 100) {
-    setTopLevelJobProperties(context, 'beam', defaultBranch, defaultTimeout)
+                                           String branch = 'master',
+                                           int timeout = 100,
+                                           String jenkinsExecutorLabel = 'beam') {
+    setTopLevelJobProperties(
+            context,
+            'beam',
+            branch,
+            jenkinsExecutorLabel,
+            timeout)
   }
 
   // Sets common top-level job properties. Accessed through one of the above
@@ -39,6 +50,7 @@ class common_job_properties {
   private static void setTopLevelJobProperties(context,
                                                String repositoryName,
                                                String defaultBranch,
+                                               String jenkinsExecutorLabel,
                                                int defaultTimeout) {
 
     // GitHub project.
@@ -49,9 +61,8 @@ class common_job_properties {
     // Set JDK version.
     context.jdk('JDK 1.8 (latest)')
 
-    // Restrict this project to run only on Jenkins executors dedicated to the
-    // Apache Beam project.
-    context.label('beam')
+    // Restrict this project to run only on Jenkins executors as specified
+    context.label(jenkinsExecutorLabel)
 
     // Discard old builds. Build records are only kept up to this number of days.
     context.logRotator {
@@ -163,8 +174,8 @@ class common_job_properties {
   }
 
   // Sets common config for Maven jobs.
-  static void setMavenConfig(context) {
-    context.mavenInstallation('Maven 3.3.3')
+  static void setMavenConfig(context, mavenInstallation='Maven 3.3.3') {
+    context.mavenInstallation(mavenInstallation)
     context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true')
     context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS')
     // The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy
new file mode 100644
index 000000000000..f781b4eec033
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall_Windows.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java postcommit which runs maven install targeting Jenkins running on Windows.
+mavenJob('beam_PostCommit_Java_MavenInstall_Windows') {
+  description('Runs postcommit tests on Windows for the Java SDK.')
+
+  // Execute concurrent builds if necessary.
+  concurrentBuild()
+
+  // Set common parameters. Note the usage of the Windows label to filter Jenkins executors.
+  common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 100, 'Windows')
+
+  // Set Maven parameters. Note the usage of the Windows Maven installation
+  common_job_properties.setMavenConfig(delegate, 'Maven 3.3.3 (Windows)')
+
+  // Sets that this is a PostCommit job.
+  common_job_properties.setPostCommit(delegate, '0 */6 * * *', false)
+
+  // Allows triggering this build against pull requests.
+  common_job_properties.enablePhraseTriggeringFromPullRequest(
+          delegate,
+          'Java SDK Windows PostCommit Tests',
+          'Run Java Windows PostCommit')
+
+  // Maven goals for this job.
+  goals('-B -e -Prelease,direct-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
+}
diff --git a/sdks/common/runner-api/pom.xml b/sdks/common/runner-api/pom.xml
index 3f550bfcbb0f..f5536a76a8ce 100644
--- a/sdks/common/runner-api/pom.xml
+++ b/sdks/common/runner-api/pom.xml
@@ -65,14 +65,11 @@
         <artifactId>protobuf-maven-plugin</artifactId>
         <configuration>
           <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
-          <pluginId>grpc-java</pluginId>
-          <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
         </configuration>
         <executions>
           <execution>
             <goals>
               <goal>compile</goal>
-              <goal>compile-custom</goal>
             </goals>
           </execution>
         </executions>

From b3f74044436f4b4a92f3135411b5d5093437ec3e Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 15 May 2017 14:34:45 -0700
Subject: [PATCH 241/387] Fix documentation for the shard_template_name

---
 sdks/python/apache_beam/io/avroio.py     | 13 ++++++-------
 sdks/python/apache_beam/io/textio.py     | 13 ++++++-------
 sdks/python/apache_beam/io/tfrecordio.py | 15 +++++++--------
 3 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index e02e1f7e893f..7df9983b1f25 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -312,13 +312,12 @@ def __init__(self,
         the performance of a pipeline.  Setting this value is not recommended
         unless you require a specific number of output files.
       shard_name_template: A template string containing placeholders for
-        the shard number and shard count. Currently only '' and
-        '-SSSSS-of-NNNNN' are patterns accepted by the service.
-        When constructing a filename for a particular shard number, the
-        upper-case letters 'S' and 'N' are replaced with the 0-padded shard
-        number and shard count respectively.  This argument can be '' in which
-        case it behaves as if num_shards was set to 1 and only one file will be
-        generated. The default pattern used is '-SSSSS-of-NNNNN'.
+        the shard number and shard count. When constructing a filename for a
+        particular shard number, the upper-case letters 'S' and 'N' are
+        replaced with the 0-padded shard number and shard count respectively.
+        This argument can be '' in which case it behaves as if num_shards was
+        set to 1 and only one file will be generated. The default pattern used
+        is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
       mime_type: The MIME type to use for the produced files, if the filesystem
         supports specifying MIME types.
 
diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py
index eeefaf6fd94e..60e1512b0265 100644
--- a/sdks/python/apache_beam/io/textio.py
+++ b/sdks/python/apache_beam/io/textio.py
@@ -291,13 +291,12 @@ def __init__(self,
         the performance of a pipeline.  Setting this value is not recommended
         unless you require a specific number of output files.
       shard_name_template: A template string containing placeholders for
-        the shard number and shard count. Currently only '' and
-        '-SSSSS-of-NNNNN' are patterns accepted by the service.
-        When constructing a filename for a particular shard number, the
-        upper-case letters 'S' and 'N' are replaced with the 0-padded shard
-        number and shard count respectively.  This argument can be '' in which
-        case it behaves as if num_shards was set to 1 and only one file will be
-        generated. The default pattern used is '-SSSSS-of-NNNNN'.
+        the shard number and shard count. When constructing a filename for a
+        particular shard number, the upper-case letters 'S' and 'N' are
+        replaced with the 0-padded shard number and shard count respectively.
+        This argument can be '' in which case it behaves as if num_shards was
+        set to 1 and only one file will be generated. The default pattern used
+        is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
       coder: Coder used to encode each line.
       compression_type: Used to handle compressed output files. Typical value
         is CompressionTypes.AUTO, in which case the final file path's
diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py
index a8cd1ce25cc5..d7eb932cd655 100644
--- a/sdks/python/apache_beam/io/tfrecordio.py
+++ b/sdks/python/apache_beam/io/tfrecordio.py
@@ -242,7 +242,7 @@ def __init__(self,
                coder=coders.BytesCoder(),
                file_name_suffix='',
                num_shards=0,
-               shard_name_template=filebasedsink.DEFAULT_SHARD_NAME_TEMPLATE,
+               shard_name_template=None,
                compression_type=CompressionTypes.AUTO,
                **kwargs):
     """Initialize WriteToTFRecord transform.
@@ -256,13 +256,12 @@ def __init__(self,
       num_shards: The number of files (shards) used for output. If not set, the
         default value will be used.
       shard_name_template: A template string containing placeholders for
-        the shard number and shard count. Currently only '' and
-        '-SSSSS-of-NNNNN' are patterns allowed.
-        When constructing a filename for a particular shard number, the
-        upper-case letters 'S' and 'N' are replaced with the 0-padded shard
-        number and shard count respectively.  This argument can be '' in which
-        case it behaves as if num_shards was set to 1 and only one file will be
-        generated. The default pattern is '-SSSSS-of-NNNNN'.
+        the shard number and shard count. When constructing a filename for a
+        particular shard number, the upper-case letters 'S' and 'N' are
+        replaced with the 0-padded shard number and shard count respectively.
+        This argument can be '' in which case it behaves as if num_shards was
+        set to 1 and only one file will be generated. The default pattern used
+        is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
       compression_type: Used to handle compressed output files. Typical value
           is CompressionTypes.AUTO, in which case the file_path's extension will
           be used to detect the compression.

From f5efca0292eb9ace2d6b4895bc08e94344854336 Mon Sep 17 00:00:00 2001
From: Thomas Weise <thw@apache.org>
Date: Sat, 8 Apr 2017 13:58:48 -0700
Subject: [PATCH 242/387] ApexRunner SDF support

---
 .../apache/beam/runners/apex/ApexRunner.java  |  34 +++++
 .../translation/ApexPipelineTranslator.java   |  23 ++++
 .../apex/translation/ParDoTranslator.java     |  68 +++++++++-
 .../operators/ApexGroupByKeyOperator.java     |   4 +-
 .../operators/ApexParDoOperator.java          | 124 +++++++++++++++++-
 .../operators/ApexProcessFnOperator.java      |   8 +-
 .../operators/ApexTimerInternals.java         |  21 ++-
 .../translation/utils/ApexStateInternals.java |   4 +
 .../utils/StateInternalsProxy.java            |   7 +-
 .../operators/ApexTimerInternalsTest.java     |   8 +-
 10 files changed, 286 insertions(+), 15 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index 366308ec6642..2fd0b22363f5 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -31,6 +31,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.apex.api.EmbeddedAppLauncher;
@@ -38,9 +39,11 @@
 import org.apache.apex.api.Launcher.AppHandle;
 import org.apache.apex.api.Launcher.LaunchMode;
 import org.apache.beam.runners.apex.translation.ApexPipelineTranslator;
+import org.apache.beam.runners.core.SplittableParDo;
 import org.apache.beam.runners.core.construction.PTransformMatchers;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.PrimitiveCreate;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineRunner;
@@ -51,18 +54,23 @@
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverride;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.View.AsIterable;
 import org.apache.beam.sdk.transforms.View.AsSingleton;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PCollectionViews;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
 import org.apache.hadoop.conf.Configuration;
 
 /**
@@ -112,6 +120,10 @@ private List<PTransformOverride> getOverrides() {
             PTransformOverride.of(
                 PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
                 new StreamingCombineGloballyAsSingletonView.Factory()))
+        .add(
+            PTransformOverride.of(
+                PTransformMatchers.splittableParDoMulti(),
+                new SplittableParDoOverrideFactory<>()))
         .build();
   }
 
@@ -424,4 +436,26 @@ public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inp
     }
   }
 
+  /**
+   * A {@link PTransformOverrideFactory} that overrides a
+   * <a href="https://s.apache.org/splittable-do-fn">Splittable DoFn</a> with
+   * {@link SplittableParDo}.
+   */
+  static class SplittableParDoOverrideFactory<InputT, OutputT> implements PTransformOverrideFactory<
+      PCollection<InputT>, PCollectionTuple, MultiOutput<InputT, OutputT>> {
+    @Override
+    public PTransformReplacement<PCollection<InputT>, PCollectionTuple> getReplacementTransform(
+        AppliedPTransform<PCollection<InputT>, PCollectionTuple, MultiOutput<InputT, OutputT>>
+          transform) {
+      return PTransformReplacement.of(PTransformReplacements.getSingletonMainInput(transform),
+          new SplittableParDo<>(transform.getTransform()));
+    }
+
+    @Override
+    public Map<PValue, ReplacementOutput> mapOutputs(Map<TupleTag<?>, PValue> outputs,
+        PCollectionTuple newOutput) {
+      return ReplacementOutputs.tagged(outputs, newOutput);
+    }
+  }
+
 }
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
index fdeefc701447..b3a6d1c0bf64 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
@@ -23,7 +23,9 @@
 import java.util.Map;
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.runners.apex.ApexRunner.CreateApexPCollectionView;
+import org.apache.beam.runners.apex.translation.operators.ApexProcessFnOperator;
 import org.apache.beam.runners.apex.translation.operators.ApexReadUnboundedInputOperator;
+import org.apache.beam.runners.core.SplittableParDo;
 import org.apache.beam.runners.core.construction.PrimitiveCreate;
 import org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter;
 import org.apache.beam.sdk.Pipeline;
@@ -35,6 +37,7 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
 import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PValue;
 import org.slf4j.Logger;
@@ -60,6 +63,10 @@ public class ApexPipelineTranslator implements Pipeline.PipelineVisitor {
   static {
     // register TransformTranslators
     registerTransformTranslator(ParDo.MultiOutput.class, new ParDoTranslator<>());
+    registerTransformTranslator(SplittableParDo.ProcessElements.class,
+        new ParDoTranslator.SplittableProcessElementsTranslator());
+    registerTransformTranslator(SplittableParDo.GBKIntoKeyedWorkItems.class,
+        new GBKIntoKeyedWorkItemsTranslator());
     registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
     registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
     registerTransformTranslator(GroupByKey.class, new GroupByKeyTranslator());
@@ -174,4 +181,20 @@ public void translate(
       LOG.debug("view {}", view.getName());
     }
   }
+
+  private static class GBKIntoKeyedWorkItemsTranslator<K, InputT>
+    implements TransformTranslator<SplittableParDo.GBKIntoKeyedWorkItems<K, InputT>> {
+
+    @Override
+    public void translate(
+        SplittableParDo.GBKIntoKeyedWorkItems<K, InputT> transform, TranslationContext context) {
+      // https://issues.apache.org/jira/browse/BEAM-1850
+      ApexProcessFnOperator<KV<K, InputT>> operator = ApexProcessFnOperator.toKeyedWorkItems(
+          context.getPipelineOptions());
+      context.addOperator(operator, operator.outputPort);
+      context.addStream(context.getInput(), operator.inputPort);
+    }
+
+  }
+
 }
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
index 2e3d902e0106..9133cb66f651 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
@@ -30,11 +30,13 @@
 import java.util.Map.Entry;
 import org.apache.beam.runners.apex.ApexRunner;
 import org.apache.beam.runners.apex.translation.operators.ApexParDoOperator;
+import org.apache.beam.runners.core.SplittableParDo;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
 import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
 import org.apache.beam.sdk.values.PCollection;
@@ -83,7 +85,7 @@ public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationC
     }
 
     Map<TupleTag<?>, PValue> outputs = context.getOutputs();
-    PCollection<InputT> input = (PCollection<InputT>) context.getInput();
+    PCollection<InputT> input = context.getInput();
     List<PCollectionView<?>> sideInputs = transform.getSideInputs();
     Coder<InputT> inputCoder = input.getCoder();
     WindowedValueCoder<InputT> wvInputCoder =
@@ -130,6 +132,70 @@ public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationC
     }
   }
 
+  static class SplittableProcessElementsTranslator<InputT, OutputT,
+      RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+    implements TransformTranslator<SplittableParDo.ProcessElements<InputT, OutputT,
+      RestrictionT, TrackerT>> {
+
+    @Override
+    public void translate(
+        SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform,
+        TranslationContext context) {
+
+      Map<TupleTag<?>, PValue> outputs = context.getOutputs();
+      PCollection<InputT> input = context.getInput();
+      List<PCollectionView<?>> sideInputs = transform.getSideInputs();
+      Coder<InputT> inputCoder = input.getCoder();
+      WindowedValueCoder<InputT> wvInputCoder =
+          FullWindowedValueCoder.of(
+              inputCoder, input.getWindowingStrategy().getWindowFn().windowCoder());
+
+      @SuppressWarnings({ "rawtypes", "unchecked" })
+      DoFn<InputT, OutputT> doFn = (DoFn) transform.newProcessFn(transform.getFn());
+      ApexParDoOperator<InputT, OutputT> operator = new ApexParDoOperator<>(
+              context.getPipelineOptions(),
+              doFn,
+              transform.getMainOutputTag(),
+              transform.getAdditionalOutputTags().getAll(),
+              input.getWindowingStrategy(),
+              sideInputs,
+              wvInputCoder,
+              context.getStateBackend());
+
+      Map<PCollection<?>, OutputPort<?>> ports = Maps.newHashMapWithExpectedSize(outputs.size());
+      for (Entry<TupleTag<?>, PValue> output : outputs.entrySet()) {
+        checkArgument(
+            output.getValue() instanceof PCollection,
+            "%s %s outputs non-PCollection %s of type %s",
+            ParDo.MultiOutput.class.getSimpleName(),
+            context.getFullName(),
+            output.getValue(),
+            output.getValue().getClass().getSimpleName());
+        PCollection<?> pc = (PCollection<?>) output.getValue();
+        if (output.getKey().equals(transform.getMainOutputTag())) {
+          ports.put(pc, operator.output);
+        } else {
+          int portIndex = 0;
+          for (TupleTag<?> tag : transform.getAdditionalOutputTags().getAll()) {
+            if (tag.equals(output.getKey())) {
+              ports.put(pc, operator.additionalOutputPorts[portIndex]);
+              break;
+            }
+            portIndex++;
+          }
+        }
+      }
+
+      context.addOperator(operator, ports);
+      context.addStream(context.getInput(), operator.input);
+      if (!sideInputs.isEmpty()) {
+        addSideInputs(operator.sideInput1, sideInputs, context);
+      }
+
+    }
+  }
+
+
   static void addSideInputs(
       Operator.InputPort<?> sideInputPort,
       List<PCollectionView<?>> sideInputs,
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
index 85836ad2acbf..1d48e2032d8a 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
@@ -202,7 +202,7 @@ private void processElement(WindowedValue<KV<K, V>> windowedValue) throws Except
         windowedValue.getTimestamp(),
         windowedValue.getWindows(),
         windowedValue.getPane());
-    timerInternals.setContext(kv.getKey(), this.keyCoder, this.inputWatermark);
+    timerInternals.setContext(kv.getKey(), this.keyCoder, this.inputWatermark, null);
     ReduceFnRunner<K, V, Iterable<V>, BoundedWindow> reduceFnRunner =
         newReduceFnRunner(kv.getKey());
     reduceFnRunner.processElements(Collections.singletonList(updatedWindowedValue));
@@ -211,7 +211,7 @@ private void processElement(WindowedValue<KV<K, V>> windowedValue) throws Except
 
   @Override
   public void fireTimer(K key, Collection<TimerData> timerData) {
-    timerInternals.setContext(key, keyCoder, inputWatermark);
+    timerInternals.setContext(key, keyCoder, inputWatermark, null);
     ReduceFnRunner<K, V, Iterable<V>, BoundedWindow> reduceFnRunner = newReduceFnRunner(key);
     try {
       reduceFnRunner.onTimers(timerData);
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
index 8c516b1eb9ca..7fee0d5fcdf4 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
@@ -17,6 +17,9 @@
  */
 package org.apache.beam.runners.apex.translation.operators;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
 import com.datatorrent.api.Context.OperatorContext;
 import com.datatorrent.api.DefaultInputPort;
 import com.datatorrent.api.DefaultOutputPort;
@@ -28,8 +31,10 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executors;
 import org.apache.beam.runners.apex.ApexPipelineOptions;
 import org.apache.beam.runners.apex.ApexRunner;
 import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
@@ -44,20 +49,33 @@
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItemCoder;
 import org.apache.beam.runners.core.NullSideInputReader;
+import org.apache.beam.runners.core.OutputAndTimeBoundedSplittableProcessElementInvoker;
+import org.apache.beam.runners.core.OutputWindowedValue;
 import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
 import org.apache.beam.runners.core.SideInputHandler;
 import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.SplittableParDo;
 import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsFactory;
+import org.apache.beam.runners.core.StateNamespace;
+import org.apache.beam.runners.core.StateNamespaces.WindowNamespace;
 import org.apache.beam.runners.core.StatefulDoFnRunner;
 import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.TimerInternalsFactory;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
@@ -65,6 +83,7 @@
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.WindowingStrategy;
+import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -72,7 +91,8 @@
 /**
  * Apex operator for Beam {@link DoFn}.
  */
-public class ApexParDoOperator<InputT, OutputT> extends BaseOperator implements OutputManager {
+public class ApexParDoOperator<InputT, OutputT> extends BaseOperator implements OutputManager,
+    ApexTimerInternals.TimerProcessor<Object> {
   private static final Logger LOG = LoggerFactory.getLogger(ApexParDoOperator.class);
   private boolean traceTuples = true;
 
@@ -139,6 +159,14 @@ public ApexParDoOperator(
     TimerInternals.TimerDataCoder timerCoder =
         TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
     this.currentKeyTimerInternals = new ApexTimerInternals<>(timerCoder);
+
+    if (doFn instanceof SplittableParDo.ProcessFn) {
+      // we know that it is keyed on String
+      Coder<?> keyCoder = StringUtf8Coder.of();
+      this.currentKeyStateInternals = new StateInternalsProxy<>(
+          stateBackend.newStateInternalsFactory(keyCoder));
+    }
+
   }
 
   @SuppressWarnings("unused") // for Kryo
@@ -272,7 +300,10 @@ private Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedVal
           keyCoder = kwiCoder.getKeyCoder();
         }
         ((StateInternalsProxy) currentKeyStateInternals).setKey(key);
-        currentKeyTimerInternals.setContext(key, keyCoder, new Instant(this.currentInputWatermark));
+        currentKeyTimerInternals.setContext(key, keyCoder,
+            new Instant(this.currentInputWatermark),
+            new Instant(this.currentOutputWatermark)
+            );
       }
       Iterable<WindowedValue<InputT>> pushedBack = pushbackDoFnRunner
           .processElementInReadyWindows(elem);
@@ -286,9 +317,47 @@ private Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedVal
     }
   }
 
+  @Override
+  public void fireTimer(Object key, Collection<TimerData> timerDataSet) {
+    pushbackDoFnRunner.startBundle();
+    @SuppressWarnings("unchecked")
+    Coder<Object> keyCoder = (Coder) currentKeyStateInternals.getKeyCoder();
+    ((StateInternalsProxy) currentKeyStateInternals).setKey(key);
+    currentKeyTimerInternals.setContext(key, keyCoder, new Instant(this.currentInputWatermark),
+        new Instant(this.currentOutputWatermark));
+    for (TimerData timerData : timerDataSet) {
+      StateNamespace namespace = timerData.getNamespace();
+      checkArgument(namespace instanceof WindowNamespace);
+      BoundedWindow window = ((WindowNamespace<?>) namespace).getWindow();
+      pushbackDoFnRunner.onTimer(timerData.getTimerId(), window,
+          timerData.getTimestamp(), timerData.getDomain());
+    }
+    pushbackDoFnRunner.finishBundle();
+  }
+
   private void processWatermark(ApexStreamTuple.WatermarkTuple<?> mark) {
     this.currentInputWatermark = mark.getTimestamp();
-
+    long minEventTimeTimer = currentKeyTimerInternals.fireReadyTimers(
+        this.currentInputWatermark,
+        this, TimeDomain.EVENT_TIME);
+
+    checkState(minEventTimeTimer >= currentInputWatermark,
+        "Event time timer processing generates new timer(s) behind watermark.");
+    //LOG.info("Processing time timer {} registered behind watermark {}", minProcessingTimeTimer,
+    //    currentInputWatermark);
+
+    // TODO: is this the right way to trigger processing time timers?
+    // drain all timers below current watermark, including those that result from firing
+    long minProcessingTimeTimer = Long.MIN_VALUE;
+    while (minProcessingTimeTimer < currentInputWatermark) {
+      minProcessingTimeTimer = currentKeyTimerInternals.fireReadyTimers(
+        this.currentInputWatermark,
+        this, TimeDomain.PROCESSING_TIME);
+      if (minProcessingTimeTimer < currentInputWatermark) {
+        LOG.info("Processing time timer {} registered behind watermark {}", minProcessingTimeTimer,
+            currentInputWatermark);
+      }
+    }
     if (sideInputs.isEmpty()) {
       if (traceTuples) {
         LOG.debug("\nemitting watermark {}\n", mark);
@@ -376,6 +445,52 @@ public TimerInternals timerInternals() {
     pushbackDoFnRunner =
         SimplePushbackSideInputDoFnRunner.create(doFnRunner, sideInputs, sideInputHandler);
 
+    if (doFn instanceof SplittableParDo.ProcessFn) {
+
+      @SuppressWarnings("unchecked")
+      StateInternalsFactory<String> stateInternalsFactory =
+          (StateInternalsFactory<String>) this.currentKeyStateInternals.getFactory();
+
+      @SuppressWarnings({ "rawtypes", "unchecked" })
+      SplittableParDo.ProcessFn<InputT, OutputT, Object, RestrictionTracker<Object>>
+        splittableDoFn = (SplittableParDo.ProcessFn) doFn;
+      splittableDoFn.setStateInternalsFactory(stateInternalsFactory);
+      TimerInternalsFactory<String> timerInternalsFactory = new TimerInternalsFactory<String>() {
+         @Override
+         public TimerInternals timerInternalsForKey(String key) {
+           return currentKeyTimerInternals;
+          }
+        };
+      splittableDoFn.setTimerInternalsFactory(timerInternalsFactory);
+      splittableDoFn.setProcessElementInvoker(
+          new OutputAndTimeBoundedSplittableProcessElementInvoker<>(
+              doFn,
+              pipelineOptions.get(),
+              new OutputWindowedValue<OutputT>() {
+                @Override
+                public void outputWindowedValue(
+                    OutputT output,
+                    Instant timestamp,
+                    Collection<? extends BoundedWindow> windows,
+                    PaneInfo pane) {
+                  output(
+                      mainOutputTag,
+                      WindowedValue.of(output, timestamp, windows, pane));
+                }
+
+                @Override
+                public <AdditionalOutputT> void outputWindowedValue(TupleTag<AdditionalOutputT> tag,
+                    AdditionalOutputT output, Instant timestamp,
+                    Collection<? extends BoundedWindow> windows, PaneInfo pane) {
+                  output(tag, WindowedValue.of(output, timestamp, windows, pane));
+                }
+              },
+              sideInputReader,
+              Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory()),
+              10000,
+              Duration.standardSeconds(10)));
+    }
+
   }
 
   @Override
@@ -390,6 +505,9 @@ public void beginWindow(long windowId) {
 
   @Override
   public void endWindow() {
+    currentKeyTimerInternals.fireReadyTimers(
+        currentKeyTimerInternals.currentProcessingTime().getMillis(),
+        this, TimeDomain.PROCESSING_TIME);
   }
 
   private static class LongMin {
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexProcessFnOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexProcessFnOperator.java
index 835c9e0cdec8..9f4e6c5b13ae 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexProcessFnOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexProcessFnOperator.java
@@ -91,7 +91,13 @@ void process(ApexStreamTuple<WindowedValue<InputT>> input,
   /**
    * Convert {@link KV} into {@link KeyedWorkItem}s.
    */
-  public static class ToKeyedWorkItems<K, V> implements ApexOperatorFn<KV<K, V>> {
+  public static <K, V> ApexProcessFnOperator<KV<K, V>> toKeyedWorkItems(
+      ApexPipelineOptions options) {
+    ApexOperatorFn<KV<K, V>> fn = new ToKeyedWorkItems<>();
+    return new ApexProcessFnOperator<KV<K, V>>(fn, options.isTupleTracingEnabled());
+  }
+
+  private static class ToKeyedWorkItems<K, V> implements ApexOperatorFn<KV<K, V>> {
     @Override
     public final void process(ApexStreamTuple<WindowedValue<KV<K, V>>> tuple,
         OutputEmitter<ApexStreamTuple<? extends WindowedValue<?>>> outputEmitter) {
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java
index 1eb224c266a7..0a0dd5086d82 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternals.java
@@ -55,6 +55,7 @@ class ApexTimerInternals<K> implements TimerInternals, Serializable {
 
   private transient K currentKey;
   private transient Instant currentInputWatermark;
+  private transient Instant currentOutputWatermark;
   private transient Coder<K> keyCoder;
 
   public ApexTimerInternals(TimerDataCoder timerDataCoder) {
@@ -62,10 +63,12 @@ public ApexTimerInternals(TimerDataCoder timerDataCoder) {
     this.processingTimeTimers = new TimerSet(timerDataCoder);
   }
 
-  public void setContext(K key, Coder<K> keyCoder, Instant inputWatermark) {
+  public void setContext(K key, Coder<K> keyCoder, Instant inputWatermark,
+      Instant outputWatermark) {
     this.currentKey = key;
     this.keyCoder = keyCoder;
     this.currentInputWatermark = inputWatermark;
+    this.currentOutputWatermark = outputWatermark;
   }
 
   @VisibleForTesting
@@ -118,7 +121,7 @@ public Instant currentInputWatermarkTime() {
 
   @Override
   public Instant currentOutputWatermarkTime() {
-    return null;
+    return currentOutputWatermark;
   }
 
   public interface TimerProcessor<K> {
@@ -128,11 +131,19 @@ public interface TimerProcessor<K> {
   /**
    * Fire the timers that are ready. These are the timers
    * that are registered to be triggered at a time before the current time.
+   * Timer processing may register new timers, which can cause the returned
+   * timestamp to be before the the current time. The caller may repeat
+   * the call until such backdated timers are cleared.
+   * @return minimum timestamp of registered timers.
    */
-  public void fireReadyTimers(long currentTime,
+  public long fireReadyTimers(long currentTime,
       TimerProcessor<K> timerProcessor, TimeDomain timeDomain) {
     TimerSet timers = getTimerSet(timeDomain);
 
+    // move minTimestamp first,
+    // timer additions that result from firing may modify it
+    timers.minTimestamp = currentTime;
+
     // we keep the timers to return in a different list and launch them later
     // because we cannot prevent a trigger from registering another timer,
     // which would lead to concurrent modification exception.
@@ -173,6 +184,8 @@ public void fireReadyTimers(long currentTime,
         }
       }
     }
+
+    return timers.minTimestamp;
   }
 
   private Slice getKeyBytes(K key) {
@@ -186,6 +199,7 @@ private Slice getKeyBytes(K key) {
   protected static class TimerSet implements Serializable {
     private final Map<Slice, Set<Slice>> activeTimers = new HashMap<>();
     private final TimerDataCoder timerDataCoder;
+    private long minTimestamp = Long.MAX_VALUE;
 
     protected TimerSet(TimerDataCoder timerDataCoder) {
       this.timerDataCoder = timerDataCoder;
@@ -205,6 +219,7 @@ public void addTimer(Slice keyBytes, TimerData timer) {
       }
 
       activeTimers.put(keyBytes, timersForKey);
+      this.minTimestamp = Math.min(minTimestamp, timer.getTimestamp().getMillis());
     }
 
     public void deleteTimer(Slice keyBytes, StateNamespace namespace, String timerId) {
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
index eeea6d144474..18ea8e4d5bbd 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/ApexStateInternals.java
@@ -438,6 +438,10 @@ private ApexStateInternalsFactory(Coder<K> keyCoder) {
       this.keyCoder = keyCoder;
     }
 
+    public Coder<K> getKeyCoder() {
+      return this.keyCoder;
+    }
+
     @Override
     public ApexStateInternals<K> stateInternalsForKey(K key) {
       final Slice keyBytes;
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java
index ccf7e43e536c..b652c68795ea 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/StateInternalsProxy.java
@@ -24,6 +24,7 @@
 import org.apache.beam.runners.core.StateInternalsFactory;
 import org.apache.beam.runners.core.StateNamespace;
 import org.apache.beam.runners.core.StateTag;
+import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.state.State;
 import org.apache.beam.sdk.state.StateContext;
 
@@ -34,7 +35,7 @@
 @DefaultSerializer(JavaSerializer.class)
 public class StateInternalsProxy<K> implements StateInternals, Serializable {
 
-  private final StateInternalsFactory<K> factory;
+  private final ApexStateInternals.ApexStateInternalsFactory<K> factory;
   private transient K currentKey;
 
   public StateInternalsProxy(ApexStateInternals.ApexStateInternalsFactory<K> factory) {
@@ -45,6 +46,10 @@ public StateInternalsFactory<K> getFactory() {
     return this.factory;
   }
 
+  public Coder<K> getKeyCoder() {
+    return factory.getKeyCoder();
+  }
+
   public void setKey(K key) {
     currentKey = key;
   }
diff --git a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java
index 7b52223e5478..ba1c801be1b1 100644
--- a/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java
+++ b/runners/apex/src/test/java/org/apache/beam/runners/apex/translation/operators/ApexTimerInternalsTest.java
@@ -60,7 +60,7 @@ public void fireTimer(String key, Collection<TimerData> timerData) {
     Instant instant2 = new Instant(2);
 
     ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder);
-    timerInternals.setContext(key1, StringUtf8Coder.of(), Instant.now());
+    timerInternals.setContext(key1, StringUtf8Coder.of(), Instant.now(), null);
 
     TimerData timerData0 = TimerData.of("timerData0", StateNamespaces.global(),
         instant0, TimeDomain.EVENT_TIME);
@@ -98,7 +98,7 @@ public void testDeleteTimer() {
     Instant instant1 = new Instant(1);
 
     ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder);
-    timerInternals.setContext(key1, StringUtf8Coder.of(), Instant.now());
+    timerInternals.setContext(key1, StringUtf8Coder.of(), Instant.now(), null);
 
     TimerData timerData0 = TimerData.of("timerData0", StateNamespaces.global(),
         instant0, TimeDomain.EVENT_TIME);
@@ -133,11 +133,11 @@ public void testSerialization() {
         new Instant(0), TimeDomain.EVENT_TIME);
     String key = "key";
     ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder);
-    timerInternals.setContext(key, StringUtf8Coder.of(), Instant.now());
+    timerInternals.setContext(key, StringUtf8Coder.of(), Instant.now(), null);
     timerInternals.setTimer(timerData);
     ApexTimerInternals<String> cloned;
     assertNotNull("Serialization", cloned = KryoCloneUtils.cloneObject(timerInternals));
-    cloned.setContext(key, StringUtf8Coder.of(), Instant.now());
+    cloned.setContext(key, StringUtf8Coder.of(), Instant.now(), null);
     Map<?, Set<Slice>> timers = cloned.getTimerSet(TimeDomain.EVENT_TIME).getMap();
     assertEquals(1, timers.size());
   }

From 0a17c3da5a249dbe4408c5dce7e1825897865e1e Mon Sep 17 00:00:00 2001
From: Vikas Kedigehalli <vikasrk@google.com>
Date: Mon, 15 May 2017 15:50:25 -0700
Subject: [PATCH 243/387] Use built-in cmp python function in comparing
 datastore paths

---
 .../io/gcp/datastore/v1/fake_datastore.py     |  8 +++-
 .../apache_beam/io/gcp/datastore/v1/helper.py | 14 ++-----
 .../gcp/datastore/v1/query_splitter_test.py   | 37 +++++++++++--------
 3 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py b/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
index 23325793c15c..aa3780558d7f 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/fake_datastore.py
@@ -90,13 +90,17 @@ def create_response(entities, end_cursor, finish):
   return resp
 
 
-def create_entities(count):
+def create_entities(count, id_or_name=False):
   """Creates a list of entities with random keys."""
   entities = []
 
   for _ in range(count):
     entity_result = query_pb2.EntityResult()
-    entity_result.entity.key.path.add().name = str(uuid.uuid4())
+    if id_or_name:
+      entity_result.entity.key.path.add().id = (
+          uuid.uuid4().int & ((1 << 63) - 1))
+    else:
+      entity_result.entity.key.path.add().name = str(uuid.uuid4())
     entities.append(entity_result)
 
   return entities
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
index 9e2c0531e540..f977536f32aa 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
@@ -80,7 +80,7 @@ def compare_path(p1, p2):
   3. If no `id` is defined for both paths, then their `names` are compared.
   """
 
-  result = str_compare(p1.kind, p2.kind)
+  result = cmp(p1.kind, p2.kind)
   if result != 0:
     return result
 
@@ -88,20 +88,12 @@ def compare_path(p1, p2):
     if not p2.HasField('id'):
       return -1
 
-    return p1.id - p2.id
+    return cmp(p1.id, p2.id)
 
   if p2.HasField('id'):
     return 1
 
-  return str_compare(p1.name, p2.name)
-
-
-def str_compare(s1, s2):
-  if s1 == s2:
-    return 0
-  elif s1 < s2:
-    return -1
-  return 1
+  return cmp(p1.name, p2.name)
 
 
 def get_datastore(project):
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py
index b7b054f382bd..52f25facd058 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py
@@ -157,27 +157,32 @@ def check_get_splits(self, query, num_splits, num_entities, batch_size):
       batch_size: the number of entities returned by fake datastore in one req.
     """
 
-    entities = fake_datastore.create_entities(num_entities)
-    mock_datastore = MagicMock()
-    # Assign a fake run_query method as a side_effect to the mock.
-    mock_datastore.run_query.side_effect = \
-        fake_datastore.create_run_query(entities, batch_size)
+    # Test for both random long ids and string ids.
+    id_or_name = [True, False]
 
-    split_queries = query_splitter.get_splits(mock_datastore, query, num_splits)
+    for id_type in id_or_name:
+      entities = fake_datastore.create_entities(num_entities, id_type)
+      mock_datastore = MagicMock()
+      # Assign a fake run_query method as a side_effect to the mock.
+      mock_datastore.run_query.side_effect = \
+          fake_datastore.create_run_query(entities, batch_size)
 
-    # if request num_splits is greater than num_entities, the best it can
-    # do is one entity per split.
-    expected_num_splits = min(num_splits, num_entities + 1)
-    self.assertEqual(len(split_queries), expected_num_splits)
+      split_queries = query_splitter.get_splits(
+          mock_datastore, query, num_splits)
 
-    expected_requests = QuerySplitterTest.create_scatter_requests(
-        query, num_splits, batch_size, num_entities)
+      # if request num_splits is greater than num_entities, the best it can
+      # do is one entity per split.
+      expected_num_splits = min(num_splits, num_entities + 1)
+      self.assertEqual(len(split_queries), expected_num_splits)
 
-    expected_calls = []
-    for req in expected_requests:
-      expected_calls.append(call(req))
+      expected_requests = QuerySplitterTest.create_scatter_requests(
+          query, num_splits, batch_size, num_entities)
 
-    self.assertEqual(expected_calls, mock_datastore.run_query.call_args_list)
+      expected_calls = []
+      for req in expected_requests:
+        expected_calls.append(call(req))
+
+      self.assertEqual(expected_calls, mock_datastore.run_query.call_args_list)
 
   @staticmethod
   def create_scatter_requests(query, num_splits, batch_size, num_entities):

From 902f27a34cf65923823d10c0edf11ba27e883c30 Mon Sep 17 00:00:00 2001
From: Pablo <pabloem@google.com>
Date: Mon, 15 May 2017 15:47:18 -0700
Subject: [PATCH 244/387] Adding support for subnetwork in Python
 Pipelineoptions

---
 .../apache_beam/options/pipeline_options.py   | 11 +++++++++-
 .../runners/dataflow/internal/apiclient.py    |  2 ++
 .../dataflow/internal/apiclient_test.py       | 21 +++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py
index 983d128fd2bf..777926ab5297 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -169,7 +169,7 @@ def from_dictionary(cls, options):
     """Returns a PipelineOptions from a dictionary of arguments.
 
     Args:
-      options: Dictinary of argument value pairs.
+      options: Dictionary of argument value pairs.
 
     Returns:
       A PipelineOptions object representing the given arguments.
@@ -454,6 +454,15 @@ def _add_argparse_args(cls, parser):
         help=(
             'GCE network for launching workers. Default is up to the Dataflow '
             'service.'))
+    parser.add_argument(
+        '--subnetwork',
+        default=None,
+        help=(
+            'GCE subnetwork for launching workers. Default is up to the '
+            'Dataflow service. Expected format is '
+            'regions/REGION/subnetworks/SUBNETWORK or the fully qualified '
+            'subnetwork name. For more information, see '
+            'https://cloud.google.com/compute/docs/vpc/'))
     parser.add_argument(
         '--worker_harness_container_image',
         default=None,
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index ea4959322f1e..bfdd5e44da38 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -199,6 +199,8 @@ def __init__(self, packages, options, environment_version):
       pool.zone = self.worker_options.zone
     if self.worker_options.network:
       pool.network = self.worker_options.network
+    if self.worker_options.subnetwork:
+      pool.subnetwork = self.worker_options.subnetwork
     if self.worker_options.worker_harness_container_image:
       pool.workerHarnessContainerImage = (
           self.worker_options.worker_harness_container_image)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index 6ed1fb4a0322..67cf77fcb93f 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -44,6 +44,27 @@ def test_create_application_client(self):
         pipeline_options,
         DataflowRunner.BATCH_ENVIRONMENT_MAJOR_VERSION)
 
+  def test_set_network(self):
+    pipeline_options = PipelineOptions(
+        ['--network', 'anetworkname',
+         '--temp_location', 'gs://any-location/temp'])
+    env = apiclient.Environment([], #packages
+                                pipeline_options,
+                                '2.0.0') #any environment version
+    self.assertEqual(env.proto.workerPools[0].network,
+                     'anetworkname')
+
+  def test_set_subnetwork(self):
+    pipeline_options = PipelineOptions(
+        ['--subnetwork', '/regions/MY/subnetworks/SUBNETWORK',
+         '--temp_location', 'gs://any-location/temp'])
+
+    env = apiclient.Environment([], #packages
+                                pipeline_options,
+                                '2.0.0') #any environment version
+    self.assertEqual(env.proto.workerPools[0].subnetwork,
+                     '/regions/MY/subnetworks/SUBNETWORK')
+
   def test_invalid_default_job_name(self):
     # Regexp for job names in dataflow.
     regexp = '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'

From 9d86bcd0e199b34220feaa583d933c5e7b3215a8 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 9 May 2017 16:09:25 -0700
Subject: [PATCH 245/387] Improve Pruning performed by the
 DirectRunnerApiSurfaceTest

Prune the parts of the Core SDK that must be exposed as part of the API
surface. The remainder should not be exposed accidentally.
---
 .../direct/DirectRunnerApiSurfaceTest.java      | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java
index 9928cb0d36b7..631349fc251a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java
@@ -23,7 +23,12 @@
 
 import com.google.common.collect.ImmutableSet;
 import java.util.Set;
-import javax.annotation.Nullable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
+import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.util.ApiSurface;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -43,9 +48,17 @@ public void testDirectRunnerApiSurface() throws Exception {
     final ClassLoader thisClassLoader = getClass().getClassLoader();
     ApiSurface apiSurface =
         ApiSurface.ofPackage(thisPackage, thisClassLoader)
+            // Do not include dependencies that are required based on the known exposures. This
+            // could alternatively prune everything exposed by the public parts of the Core SDK
+            .pruningClass(Pipeline.class)
+            .pruningClass(PipelineRunner.class)
+            .pruningClass(PipelineOptions.class)
+            .pruningClass(PipelineOptionsRegistrar.class)
+            .pruningClass(PipelineOptions.DirectRunner.class)
+            .pruningClass(DisplayData.Builder.class)
+            .pruningClass(MetricResults.class)
             .pruningPattern("org[.]apache[.]beam[.].*Test.*")
             .pruningPattern("org[.]apache[.]beam[.].*IT")
-            .pruningClass(Nullable.class)
             .pruningPattern("java[.]io.*")
             .pruningPattern("java[.]lang.*")
             .pruningPattern("java[.]util.*");

From 00b4a30f20d32a29204ec67d33061c31b43995a3 Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Fri, 12 May 2017 11:04:55 -0700
Subject: [PATCH 246/387] Bump Dataflow containers to 0512

---
 runners/google-cloud-dataflow-java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 073cee8085a5..9577ceb07cc3 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170510</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170512</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>

From 6127f532b7e6ad0f13926d3d9aec17eb538108ed Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Fri, 12 May 2017 11:18:57 -0700
Subject: [PATCH 247/387] Removes OldDoFn and its kin from runners-core

---
 .../beam/runners/core/AssignWindowsDoFn.java  |  78 --
 .../beam/runners/core/DoFnAdapters.java       | 381 ---------
 .../apache/beam/runners/core/DoFnRunners.java |  27 +-
 .../GroupAlsoByWindowViaOutputBufferDoFn.java | 113 ---
 .../GroupAlsoByWindowViaWindowSetDoFn.java    |  94 ---
 .../GroupAlsoByWindowViaWindowSetNewDoFn.java |   6 +-
 ...ava => GroupAlsoByWindowsAggregators.java} |  17 +-
 .../core/LateDataDroppingDoFnRunner.java      |   3 +-
 .../org/apache/beam/runners/core/OldDoFn.java | 335 --------
 .../runners/core/SimpleOldDoFnRunner.java     | 499 ------------
 .../core/WindowingInternalsAdapters.java      |  74 --
 ...upAlsoByWindowViaOutputBufferDoFnTest.java | 109 ---
 .../core/GroupAlsoByWindowsProperties.java    | 744 ------------------
 .../apache/beam/runners/core/NoOpOldDoFn.java |  65 --
 .../apache/beam/runners/core/OldDoFnTest.java |  51 --
 .../runners/core/SimpleOldDoFnRunnerTest.java |  86 --
 .../GroupAlsoByWindowEvaluatorFactory.java    |   6 +-
 .../SparkGroupAlsoByWindowViaWindowSet.java   |   9 +-
 .../translation/SparkAssignWindowFn.java      |   4 +-
 ...arkGroupAlsoByWindowViaOutputBufferFn.java |   4 +-
 20 files changed, 19 insertions(+), 2686 deletions(-)
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/AssignWindowsDoFn.java
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java
 rename runners/core-java/src/main/java/org/apache/beam/runners/core/{GroupAlsoByWindowsDoFn.java => GroupAlsoByWindowsAggregators.java} (61%)
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java
 delete mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java
 delete mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java
 delete mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/NoOpOldDoFn.java
 delete mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/OldDoFnTest.java
 delete mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleOldDoFnRunnerTest.java

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/AssignWindowsDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/AssignWindowsDoFn.java
deleted file mode 100644
index bbf3574616bd..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/AssignWindowsDoFn.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.collect.Iterables;
-import java.util.Collection;
-import org.apache.beam.runners.core.OldDoFn.RequiresWindowAccess;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.values.PCollection;
-import org.joda.time.Instant;
-
-/**
- * {@link OldDoFn} that tags elements of a {@link PCollection} with windows, according to the
- * provided {@link WindowFn}.
- *
- * @param <T> Type of elements being windowed
- * @param <W> Window type
- */
-@SystemDoFnInternal
-public class AssignWindowsDoFn<T, W extends BoundedWindow> extends OldDoFn<T, T>
-    implements RequiresWindowAccess {
-  private WindowFn<? super T, W> fn;
-
-  public AssignWindowsDoFn(WindowFn<? super T, W> fn) {
-    this.fn =
-        checkNotNull(
-            fn,
-            "%s provided to %s cannot be null",
-            WindowFn.class.getSimpleName(),
-            AssignWindowsDoFn.class.getSimpleName());
-  }
-
-  @Override
-  @SuppressWarnings("unchecked")
-  public void processElement(final ProcessContext c) throws Exception {
-    Collection<W> windows =
-        ((WindowFn<T, W>) fn).assignWindows(
-            ((WindowFn<T, W>) fn).new AssignContext() {
-                @Override
-                public T element() {
-                  return c.element();
-                }
-
-                @Override
-                public Instant timestamp() {
-                  return c.timestamp();
-                }
-
-                @Override
-                public BoundedWindow window() {
-                  return Iterables.getOnlyElement(c.windowingInternals().windows());
-                }
-              });
-
-    c.windowingInternals()
-        .outputWindowedValue(c.element(), c.timestamp(), windows, PaneInfo.NO_FIRING);
-  }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
deleted file mode 100644
index af59a40b7518..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.io.IOException;
-import org.apache.beam.runners.core.OldDoFn.Context;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.state.State;
-import org.apache.beam.sdk.state.Timer;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
-import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
-import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
-import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
-import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
-import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-
-/**
- * Utility class containing adapters to/from {@link DoFn} and {@link OldDoFn}.
- *
- * @deprecated This class will go away when we start running {@link DoFn}'s directly (using {@link
- *     DoFnInvoker}) rather than via {@link OldDoFn}.
- */
-@Deprecated
-public class DoFnAdapters {
-  /** Should not be instantiated. */
-  private DoFnAdapters() {}
-
-  /** Creates an {@link OldDoFn} that delegates to the {@link DoFn}. */
-  @SuppressWarnings({"unchecked", "rawtypes"})
-  public static <InputT, OutputT> OldDoFn<InputT, OutputT> toOldDoFn(DoFn<InputT, OutputT> fn) {
-    DoFnSignature signature = DoFnSignatures.getSignature((Class) fn.getClass());
-    if (signature.processElement().observesWindow()) {
-      return new WindowDoFnAdapter<>(fn);
-    } else {
-      return new SimpleDoFnAdapter<>(fn);
-    }
-  }
-
-  /**
-   * Wraps a {@link DoFn} that doesn't require access to {@link BoundedWindow} as an {@link
-   * OldDoFn}.
-   */
-  private static class SimpleDoFnAdapter<InputT, OutputT> extends OldDoFn<InputT, OutputT> {
-    private final DoFn<InputT, OutputT> fn;
-    private transient DoFnInvoker<InputT, OutputT> invoker;
-
-    SimpleDoFnAdapter(DoFn<InputT, OutputT> fn) {
-      this.fn = fn;
-      this.invoker = DoFnInvokers.invokerFor(fn);
-    }
-
-    @Override
-    public void setup() throws Exception {
-      this.invoker.invokeSetup();
-    }
-
-    @Override
-    public void startBundle(Context c) throws Exception {
-      fn.prepareForProcessing();
-      invoker.invokeStartBundle(new StartBundleContextAdapter<>(fn, c));
-    }
-
-    @Override
-    public void finishBundle(Context c) throws Exception {
-      invoker.invokeFinishBundle(new FinishBundleContextAdapter<>(fn, c));
-    }
-
-    @Override
-    public void teardown() throws Exception {
-      this.invoker.invokeTeardown();
-    }
-
-    @Override
-    public void processElement(ProcessContext c) throws Exception {
-      ProcessContextAdapter<InputT, OutputT> adapter = new ProcessContextAdapter<>(fn, c);
-      invoker.invokeProcessElement(adapter);
-    }
-
-    @Override
-    public Duration getAllowedTimestampSkew() {
-      return fn.getAllowedTimestampSkew();
-    }
-
-    @Override
-    public void populateDisplayData(DisplayData.Builder builder) {
-      builder.delegate(fn);
-    }
-
-    private void readObject(java.io.ObjectInputStream in)
-        throws IOException, ClassNotFoundException {
-      in.defaultReadObject();
-      this.invoker = DoFnInvokers.invokerFor(fn);
-    }
-  }
-
-  /** Wraps a {@link DoFn} that requires access to {@link BoundedWindow} as an {@link OldDoFn}. */
-  private static class WindowDoFnAdapter<InputT, OutputT> extends SimpleDoFnAdapter<InputT, OutputT>
-      implements OldDoFn.RequiresWindowAccess {
-
-    WindowDoFnAdapter(DoFn<InputT, OutputT> fn) {
-      super(fn);
-    }
-  }
-
-  /**
-   * Wraps an {@link OldDoFn.Context} as a {@link DoFnInvoker.ArgumentProvider} inside a {@link
-   * DoFn.StartBundle} method, which means the extra context is unavailable.
-   */
-  private static class StartBundleContextAdapter<InputT, OutputT>
-      extends DoFn<InputT, OutputT>.StartBundleContext
-      implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-    private OldDoFn<InputT, OutputT>.Context context;
-
-    private StartBundleContextAdapter(DoFn<InputT, OutputT> fn, Context context) {
-      fn.super();
-      this.context = context;
-    }
-
-    public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
-    }
-
-    @Override
-    public BoundedWindow window() {
-      // The OldDoFn doesn't allow us to ask for these outside processElement, so this
-      // should be unreachable.
-      throw new UnsupportedOperationException(
-          "Can only get the window in processElement; elsewhere there is no defined window.");
-    }
-
-    @Override
-    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
-      return this;
-    }
-
-    @Override
-    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException(
-          "Can only get a FinishBundleContext in finishBundle");
-    }
-
-    @Override
-    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException(
-          "Can only get a ProcessContext in processElement");
-    }
-
-    @Override
-    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException(
-          "Timers are not supported for OldDoFn");
-    }
-
-    @Override
-    public RestrictionTracker<?> restrictionTracker() {
-      throw new UnsupportedOperationException("This is a non-splittable DoFn");
-    }
-
-    @Override
-    public State state(String stateId) {
-      throw new UnsupportedOperationException("State is not supported by this runner");
-    }
-
-    @Override
-    public Timer timer(String timerId) {
-      throw new UnsupportedOperationException("Timers are not supported by this runner");
-    }
-  }
-
-  /**
-   * Wraps an {@link OldDoFn.Context} as a {@link DoFnInvoker.ArgumentProvider} inside a {@link
-   * DoFn.FinishBundle} method, which means the extra context is unavailable.
-   */
-  private static class FinishBundleContextAdapter<InputT, OutputT>
-      extends DoFn<InputT, OutputT>.FinishBundleContext
-      implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
-    private OldDoFn<InputT, OutputT>.Context context;
-
-    private FinishBundleContextAdapter(DoFn<InputT, OutputT> fn, Context context) {
-      fn.super();
-      this.context = context;
-    }
-
-    public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
-    }
-
-    @Override
-    public BoundedWindow window() {
-      // The OldDoFn doesn't allow us to ask for these outside processElement, so this
-      // should be unreachable.
-      throw new UnsupportedOperationException(
-          "Can only get the window in processElement; elsewhere there is no defined window.");
-    }
-
-    @Override
-    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException(
-          "Can only get a StartBundleContext in startBundle");
-    }
-
-    @Override
-    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
-      return this;
-    }
-
-    @Override
-    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException(
-          "Can only get a ProcessContext in processElement");
-    }
-
-    @Override
-    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException(
-          "Timers are not supported for OldDoFn");
-    }
-
-    @Override
-    public RestrictionTracker<?> restrictionTracker() {
-      throw new UnsupportedOperationException("This is a non-splittable DoFn");
-    }
-
-    @Override
-    public State state(String stateId) {
-      throw new UnsupportedOperationException("State is not supported by this runner");
-    }
-
-    @Override
-    public Timer timer(String timerId) {
-      throw new UnsupportedOperationException("Timers are not supported by this runner");
-    }
-
-    @Override
-    public void output(
-        OutputT output, Instant timestamp, BoundedWindow window) {
-      // Not full fidelity conversion. This should be removed as soon as possible.
-      context.outputWithTimestamp(output, timestamp);
-    }
-
-    @Override
-    public <T> void output(
-        TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
-      // Not full fidelity conversion. This should be removed as soon as possible.
-      context.outputWithTimestamp(tag, output, timestamp);
-    }
-  }
-
-  /**
-   * Wraps an {@link OldDoFn.ProcessContext} as a {@link DoFnInvoker.ArgumentProvider} method.
-   */
-  private static class ProcessContextAdapter<InputT, OutputT>
-      extends DoFn<InputT, OutputT>.ProcessContext
-      implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
-    private OldDoFn<InputT, OutputT>.ProcessContext context;
-
-    private ProcessContextAdapter(
-        DoFn<InputT, OutputT> fn, OldDoFn<InputT, OutputT>.ProcessContext context) {
-      fn.super();
-      this.context = context;
-    }
-
-    @Override
-    public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
-    }
-
-    @Override
-    public <T> T sideInput(PCollectionView<T> view) {
-      return context.sideInput(view);
-    }
-
-    @Override
-    public void output(OutputT output) {
-      context.output(output);
-    }
-
-    @Override
-    public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      context.outputWithTimestamp(output, timestamp);
-    }
-
-    @Override
-    public <T> void output(TupleTag<T> tag, T output) {
-      context.output(tag, output);
-    }
-
-    @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      context.outputWithTimestamp(tag, output, timestamp);
-    }
-
-    @Override
-    public InputT element() {
-      return context.element();
-    }
-
-    @Override
-    public Instant timestamp() {
-      return context.timestamp();
-    }
-
-    @Override
-    public PaneInfo pane() {
-      return context.pane();
-    }
-
-    @Override
-    public void updateWatermark(Instant watermark) {
-      throw new UnsupportedOperationException("Only splittable DoFn's can use updateWatermark()");
-    }
-
-    @Override
-    public BoundedWindow window() {
-      return context.window();
-    }
-
-    @Override
-    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
-      return null;
-    }
-
-    @Override
-    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
-      return null;
-    }
-
-    @Override
-    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
-      return this;
-    }
-
-    @Override
-    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
-      throw new UnsupportedOperationException("Timers are not supported for OldDoFn");
-    }
-
-    @Override
-    public RestrictionTracker<?> restrictionTracker() {
-      throw new UnsupportedOperationException("This is a non-splittable DoFn");
-    }
-
-    @Override
-    public State state(String stateId) {
-      throw new UnsupportedOperationException("State is not supported by this runner");
-    }
-
-    @Override
-    public Timer timer(String timerId) {
-      throw new UnsupportedOperationException("Timers are not supported by this runner");
-    }
-  }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
index ee3aefa1054b..f3cca6fcd23b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
@@ -73,35 +73,10 @@ public static <InputT, OutputT> DoFnRunner<InputT, OutputT> simpleRunner(
         windowingStrategy);
   }
 
-  /**
-   * Returns a basic implementation of {@link DoFnRunner} that works for most {@link OldDoFn DoFns}.
-   *
-   * <p>It invokes {@link OldDoFn#processElement} for each input.
-   */
-  public static <InputT, OutputT> DoFnRunner<InputT, OutputT> simpleRunner(
-      PipelineOptions options,
-      OldDoFn<InputT, OutputT> fn,
-      SideInputReader sideInputReader,
-      OutputManager outputManager,
-      TupleTag<OutputT> mainOutputTag,
-      List<TupleTag<?>> additionalOutputTags,
-      StepContext stepContext,
-      WindowingStrategy<?, ?> windowingStrategy) {
-    return new SimpleOldDoFnRunner<>(
-        options,
-        fn,
-        sideInputReader,
-        outputManager,
-        mainOutputTag,
-        additionalOutputTags,
-        stepContext,
-        windowingStrategy);
-  }
-
   /**
    * Returns an implementation of {@link DoFnRunner} that handles late data dropping.
    *
-   * <p>It drops elements from expired windows before they reach the underlying {@link OldDoFn}.
+   * <p>It drops elements from expired windows before they reach the underlying {@link DoFn}.
    */
   public static <K, InputT, OutputT, W extends BoundedWindow>
       DoFnRunner<KeyedWorkItem<K, InputT>, KV<K, OutputT>> lateDataDroppingRunner(
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java
deleted file mode 100644
index a1605534d6a2..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFn.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.beam.runners.core.construction.Triggers;
-import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
-import org.apache.beam.runners.core.triggers.TriggerStateMachines;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.joda.time.Instant;
-
-/**
- * The default batch {@link GroupAlsoByWindowsDoFn} implementation, if no specialized "fast path"
- * implementation is applicable.
- */
-@SystemDoFnInternal
-public class GroupAlsoByWindowViaOutputBufferDoFn<K, InputT, OutputT, W extends BoundedWindow>
-    extends GroupAlsoByWindowsDoFn<K, InputT, OutputT, W> {
-
-  private final WindowingStrategy<?, W> strategy;
-  private final StateInternalsFactory<K> stateInternalsFactory;
-  private SystemReduceFn<K, InputT, ?, OutputT, W> reduceFn;
-
-  public GroupAlsoByWindowViaOutputBufferDoFn(
-      WindowingStrategy<?, W> windowingStrategy,
-      StateInternalsFactory<K> stateInternalsFactory,
-      SystemReduceFn<K, InputT, ?, OutputT, W> reduceFn) {
-    this.strategy = windowingStrategy;
-    this.reduceFn = reduceFn;
-    this.stateInternalsFactory = stateInternalsFactory;
-  }
-
-  @Override
-  public void processElement(ProcessContext c) throws Exception {
-    K key = c.element().getKey();
-    // Used with Batch, we know that all the data is available for this key. We can't use the
-    // timer manager from the context because it doesn't exist. So we create one and emulate the
-    // watermark, knowing that we have all data and it is in timestamp order.
-    InMemoryTimerInternals timerInternals = new InMemoryTimerInternals();
-    timerInternals.advanceProcessingTime(Instant.now());
-    timerInternals.advanceSynchronizedProcessingTime(Instant.now());
-    StateInternals stateInternals = stateInternalsFactory.stateInternalsForKey(key);
-
-    ReduceFnRunner<K, InputT, OutputT, W> reduceFnRunner =
-        new ReduceFnRunner<>(
-            key,
-            strategy,
-            ExecutableTriggerStateMachine.create(
-                TriggerStateMachines.stateMachineForTrigger(
-                    Triggers.toProto(strategy.getTrigger()))),
-            stateInternals,
-            timerInternals,
-            WindowingInternalsAdapters.outputWindowedValue(c.windowingInternals()),
-            WindowingInternalsAdapters.sideInputReader(c.windowingInternals()),
-            reduceFn,
-            c.getPipelineOptions());
-
-    // Process the elements.
-    reduceFnRunner.processElements(c.element().getValue());
-
-    // Finish any pending windows by advancing the input watermark to infinity.
-    timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
-
-    // Finally, advance the processing time to infinity to fire any timers.
-    timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
-    timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
-
-    fireEligibleTimers(timerInternals, reduceFnRunner);
-
-    reduceFnRunner.persist();
-  }
-
-  private void fireEligibleTimers(
-      InMemoryTimerInternals timerInternals, ReduceFnRunner<K, InputT, OutputT, W> reduceFnRunner)
-      throws Exception {
-    List<TimerInternals.TimerData> timers = new ArrayList<>();
-    while (true) {
-      TimerInternals.TimerData timer;
-      while ((timer = timerInternals.removeNextEventTimer()) != null) {
-        timers.add(timer);
-      }
-      while ((timer = timerInternals.removeNextProcessingTimer()) != null) {
-        timers.add(timer);
-      }
-      while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) {
-        timers.add(timer);
-      }
-      if (timers.isEmpty()) {
-        break;
-      }
-      reduceFnRunner.onTimers(timers);
-      timers.clear();
-    }
-  }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java
deleted file mode 100644
index 2342c529342d..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetDoFn.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import org.apache.beam.runners.core.construction.Triggers;
-import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
-import org.apache.beam.runners.core.triggers.TriggerStateMachines;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.WindowingStrategy;
-
-/**
- * A general {@link GroupAlsoByWindowsDoFn}. This delegates all of the logic to the
- * {@link ReduceFnRunner}.
- */
-@SystemDoFnInternal
-public class GroupAlsoByWindowViaWindowSetDoFn<
-        K, InputT, OutputT, W extends BoundedWindow, RinT extends KeyedWorkItem<K, InputT>>
-    extends OldDoFn<RinT, KV<K, OutputT>> {
-
-  public static <K, InputT, OutputT, W extends BoundedWindow>
-      OldDoFn<KeyedWorkItem<K, InputT>, KV<K, OutputT>> create(
-          WindowingStrategy<?, W> strategy,
-          StateInternalsFactory<K> stateInternalsFactory,
-          SystemReduceFn<K, InputT, ?, OutputT, W> reduceFn) {
-    return new GroupAlsoByWindowViaWindowSetDoFn<>(strategy, stateInternalsFactory, reduceFn);
-  }
-
-  private final WindowingStrategy<Object, W> windowingStrategy;
-  private final StateInternalsFactory<K> stateInternalsFactory;
-  private SystemReduceFn<K, InputT, ?, OutputT, W> reduceFn;
-
-  private GroupAlsoByWindowViaWindowSetDoFn(
-      WindowingStrategy<?, W> windowingStrategy,
-      StateInternalsFactory<K> stateInternalsFactory,
-      SystemReduceFn<K, InputT, ?, OutputT, W> reduceFn) {
-    @SuppressWarnings("unchecked")
-    WindowingStrategy<Object, W> noWildcard = (WindowingStrategy<Object, W>) windowingStrategy;
-    this.windowingStrategy = noWildcard;
-    this.reduceFn = reduceFn;
-    this.stateInternalsFactory = stateInternalsFactory;
-  }
-
-  @Override
-  public void processElement(ProcessContext c) throws Exception {
-    KeyedWorkItem<K, InputT> keyedWorkItem = c.element();
-
-    K key = keyedWorkItem.key();
-    TimerInternals timerInternals = c.windowingInternals().timerInternals();
-    StateInternals stateInternals = stateInternalsFactory.stateInternalsForKey(key);
-
-    ReduceFnRunner<K, InputT, OutputT, W> reduceFnRunner =
-        new ReduceFnRunner<>(
-            key,
-            windowingStrategy,
-            ExecutableTriggerStateMachine.create(
-                TriggerStateMachines.stateMachineForTrigger(
-                    Triggers.toProto(windowingStrategy.getTrigger()))),
-            stateInternals,
-            timerInternals,
-            WindowingInternalsAdapters.outputWindowedValue(c.windowingInternals()),
-            WindowingInternalsAdapters.sideInputReader(c.windowingInternals()),
-            reduceFn,
-            c.getPipelineOptions());
-
-    reduceFnRunner.processElements(keyedWorkItem.elementsIterable());
-    reduceFnRunner.onTimers(keyedWorkItem.timersIterable());
-    reduceFnRunner.persist();
-  }
-
-  public OldDoFn<KeyedWorkItem<K, InputT>, KV<K, OutputT>> asDoFn() {
-    // Safe contravariant cast
-    @SuppressWarnings("unchecked")
-    OldDoFn<KeyedWorkItem<K, InputT>, KV<K, OutputT>> asFn =
-        (OldDoFn<KeyedWorkItem<K, InputT>, KV<K, OutputT>>) this;
-    return asFn;
-  }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
index 5b82d1fdfda4..744d1625667d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
@@ -32,7 +32,7 @@
 import org.joda.time.Instant;
 
 /**
- * A general {@link GroupAlsoByWindowsDoFn}. This delegates all of the logic to the
+ * A general {@link GroupAlsoByWindowsAggregators}. This delegates all of the logic to the
  * {@link ReduceFnRunner}.
  */
 @SystemDoFnInternal
@@ -134,8 +134,4 @@ public void processElement(ProcessContext c) throws Exception {
     reduceFnRunner.onTimers(keyedWorkItem.timersIterable());
     reduceFnRunner.persist();
   }
-
-  public OldDoFn<KeyedWorkItem<K, InputT>, KV<K, OutputT>> asDoFn() {
-    throw new RuntimeException("Not implement!");
-  }
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowsDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowsAggregators.java
similarity index 61%
rename from runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowsDoFn.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowsAggregators.java
index 2bd9ee0c6d10..8d96257ce8ac 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowsDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowsAggregators.java
@@ -17,23 +17,10 @@
  */
 package org.apache.beam.runners.core;
 
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-
 /**
- * {@link OldDoFn} that merges windows and groups elements in those windows, optionally
- * combining values.
- *
- * @param <K> key type
- * @param <InputT> input value element type
- * @param <OutputT> output value element type
- * @param <W> window type
+ * Common aggregator names for {@link GroupByKeyViaGroupByKeyOnly.GroupAlsoByWindow}.
  */
-@SystemDoFnInternal
-public abstract class GroupAlsoByWindowsDoFn<K, InputT, OutputT, W extends BoundedWindow>
-    extends OldDoFn<KV<K, Iterable<WindowedValue<InputT>>>, KV<K, OutputT>> {
+public abstract class GroupAlsoByWindowsAggregators {
   public static final String DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER = "DroppedDueToClosedWindow";
   public static final String DROPPED_DUE_TO_LATENESS_COUNTER = "DroppedDueToLateness";
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
index 570f52463666..1cf150973f7a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java
@@ -24,6 +24,7 @@
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowTracing;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -33,7 +34,7 @@
 
 /**
  * A customized {@link DoFnRunner} that handles late data dropping for
- * a {@link KeyedWorkItem} input {@link OldDoFn}.
+ * a {@link KeyedWorkItem} input {@link DoFn}.
  *
  * <p>It expands windows before checking data lateness.
  *
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java
deleted file mode 100644
index 41bb598795e6..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OldDoFn.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.io.Serializable;
-import org.apache.beam.sdk.PipelineRunner;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFnTester;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.display.HasDisplayData;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-
-/**
- * The argument to {@link ParDo} providing the code to use to process
- * elements of the input
- * {@link org.apache.beam.sdk.values.PCollection}.
- *
- * <p>See {@link ParDo} for more explanation, examples of use, and
- * discussion of constraints on {@code OldDoFn}s, including their
- * serializability, lack of access to global shared mutable state,
- * requirements for failure tolerance, and benefits of optimization.
- *
- * <p>{@code OldDoFn}s can be tested in the context of a particular
- * {@code Pipeline} by running that {@code Pipeline} on sample input
- * and then checking its output.  Unit testing of a {@code OldDoFn},
- * separately from any {@code ParDo} transform or {@code Pipeline},
- * can be done via the {@link DoFnTester} harness.
- *
- * <p>{@link DoFn} (currently experimental) offers an alternative
- * mechanism for accessing {@link ProcessContext#window()} without the need
- * to implement {@link RequiresWindowAccess}.
- *
- * <p>See also {@link #processElement} for details on implementing the transformation
- * from {@code InputT} to {@code OutputT}.
- *
- * @param <InputT> the type of the (main) input elements
- * @param <OutputT> the type of the (main) output elements
- * @deprecated Uses of {@link OldDoFn} should be replaced by the new {@link DoFn}.
- */
-@Deprecated
-public abstract class OldDoFn<InputT, OutputT> implements Serializable, HasDisplayData {
-  /**
-   * Information accessible to all methods in this {@code OldDoFn}.
-   * Used primarily to output elements.
-   */
-  public abstract class Context {
-
-    /**
-     * Returns the {@code PipelineOptions} specified with the
-     * {@link PipelineRunner}
-     * invoking this {@code OldDoFn}.  The {@code PipelineOptions} will
-     * be the default running via {@link DoFnTester}.
-     */
-    public abstract PipelineOptions getPipelineOptions();
-
-    /**
-     * Adds the given element to the main output {@code PCollection}.
-     *
-     * <p>Once passed to {@code output} the element should be considered
-     * immutable and not be modified in any way. It may be cached or retained
-     * by a Beam runner or later steps in the pipeline, or used in
-     * other unspecified ways.
-     *
-     * <p>If invoked from {@link OldDoFn#processElement processElement}, the output
-     * element will have the same timestamp and be in the same windows
-     * as the input element passed to {@link OldDoFn#processElement processElement}.
-     *
-     * <p>If invoked from {@link #startBundle startBundle} or {@link #finishBundle finishBundle},
-     * this will attempt to use the
-     * {@link org.apache.beam.sdk.transforms.windowing.WindowFn}
-     * of the input {@code PCollection} to determine what windows the element
-     * should be in, throwing an exception if the {@code WindowFn} attempts
-     * to access any information about the input element. The output element
-     * will have a timestamp of negative infinity.
-     */
-    public abstract void output(OutputT output);
-
-    /**
-     * Adds the given element to the main output {@code PCollection},
-     * with the given timestamp.
-     *
-     * <p>Once passed to {@code outputWithTimestamp} the element should not be
-     * modified in any way.
-     *
-     * <p>If invoked from {@link OldDoFn#processElement processElement}, the timestamp
-     * must not be older than the input element's timestamp minus
-     * {@link OldDoFn#getAllowedTimestampSkew getAllowedTimestampSkew}.  The output element will
-     * be in the same windows as the input element.
-     *
-     * <p>If invoked from {@link #startBundle startBundle} or {@link #finishBundle finishBundle},
-     * this will attempt to use the
-     * {@link org.apache.beam.sdk.transforms.windowing.WindowFn}
-     * of the input {@code PCollection} to determine what windows the element
-     * should be in, throwing an exception if the {@code WindowFn} attempts
-     * to access any information about the input element except for the
-     * timestamp.
-     */
-    public abstract void outputWithTimestamp(OutputT output, Instant timestamp);
-
-    /**
-     * Adds the given element to the output {@code PCollection} with the
-     * given tag.
-     *
-     * <p>Once passed to {@code output} the element should not be modified
-     * in any way.
-     *
-     * <p>The caller of {@code ParDo} uses {@link ParDo.SingleOutput#withOutputTags withOutputTags}
-     * to specify the tags of outputs that it consumes. Outputs that are not consumed, e.g., outputs
-     * for monitoring purposes only, don't necessarily need to be specified.
-     *
-     * <p>The output element will have the same timestamp and be in the same
-     * windows as the input element passed to {@link OldDoFn#processElement processElement}.
-     *
-     * <p>If invoked from {@link #startBundle startBundle} or {@link #finishBundle finishBundle},
-     * this will attempt to use the
-     * {@link org.apache.beam.sdk.transforms.windowing.WindowFn}
-     * of the input {@code PCollection} to determine what windows the element
-     * should be in, throwing an exception if the {@code WindowFn} attempts
-     * to access any information about the input element. The output element
-     * will have a timestamp of negative infinity.
-     *
-     * @see ParDo.SingleOutput#withOutputTags
-     */
-    public abstract <T> void output(TupleTag<T> tag, T output);
-
-    /**
-     * Adds the given element to the specified output {@code PCollection}, with the given timestamp.
-     *
-     * <p>Once passed to {@code outputWithTimestamp} the element should not be modified in any way.
-     *
-     * <p>If invoked from {@link OldDoFn#processElement processElement}, the timestamp must not be
-     * older than the input element's timestamp minus {@link OldDoFn#getAllowedTimestampSkew
-     * getAllowedTimestampSkew}. The output element will be in the same windows as the input
-     * element.
-     *
-     * <p>If invoked from {@link #startBundle startBundle} or {@link #finishBundle finishBundle},
-     * this will attempt to use the {@link org.apache.beam.sdk.transforms.windowing.WindowFn} of the
-     * input {@code PCollection} to determine what windows the element should be in, throwing an
-     * exception if the {@code WindowFn} attempts to access any information about the input element
-     * except for the timestamp.
-     *
-     * @see ParDo.SingleOutput#withOutputTags
-     */
-    public abstract <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp);
-  }
-
-  /**
-   * Information accessible when running {@link OldDoFn#processElement}.
-   */
-  public abstract class ProcessContext extends Context {
-
-    /**
-     * Returns the input element to be processed.
-     *
-     * <p>The element should be considered immutable. A Beam runner will not mutate the
-     * element, so it is safe to cache, etc. The element should not be mutated by any of the
-     * {@link OldDoFn} methods, because it may be cached elsewhere, retained by the runner
-     * runtime, or used in other unspecified ways.
-     */
-    public abstract InputT element();
-
-    /**
-     * Returns the value of the side input for the window corresponding to the
-     * window of the main input element.
-     *
-     * <p>See
-     * {@link WindowMappingFn#getSideInputWindow}
-     * for how this corresponding window is determined.
-     *
-     * @throws IllegalArgumentException if this is not a side input
-     * @see ParDo.SingleOutput#withSideInputs
-     */
-    public abstract <T> T sideInput(PCollectionView<T> view);
-
-    /**
-     * Returns the timestamp of the input element.
-     *
-     * <p>See {@link org.apache.beam.sdk.transforms.windowing.Window}
-     * for more information.
-     */
-    public abstract Instant timestamp();
-
-    /**
-     * Returns the window into which the input element has been assigned.
-     *
-     * <p>See {@link org.apache.beam.sdk.transforms.windowing.Window}
-     * for more information.
-     *
-     * @throws UnsupportedOperationException if this {@link OldDoFn} does
-     * not implement {@link RequiresWindowAccess}.
-     */
-    public abstract BoundedWindow window();
-
-    /**
-     * Returns information about the pane within this window into which the
-     * input element has been assigned.
-     *
-     * <p>Generally all data is in a single, uninteresting pane unless custom
-     * triggering and/or late data has been explicitly requested.
-     * See {@link org.apache.beam.sdk.transforms.windowing.Window}
-     * for more information.
-     */
-    public abstract PaneInfo pane();
-
-    /**
-     * Returns the process context to use for implementing windowing.
-     */
-    @Experimental
-    public abstract WindowingInternals<InputT, OutputT> windowingInternals();
-  }
-
-  /**
-   * Returns the allowed timestamp skew duration, which is the maximum
-   * duration that timestamps can be shifted backward in
-   * {@link OldDoFn.Context#outputWithTimestamp}.
-   *
-   * <p>The default value is {@code Duration.ZERO}, in which case
-   * timestamps can only be shifted forward to future.  For infinite
-   * skew, return {@code Duration.millis(Long.MAX_VALUE)}.
-   *
-   * <p>Note that producing an element whose timestamp is less than the
-   * current timestamp may result in late data, i.e. returning a non-zero
-   * value here does not impact watermark calculations used for firing
-   * windows.
-   *
-   * @deprecated does not interact well with the watermark.
-   */
-  @Deprecated
-  public Duration getAllowedTimestampSkew() {
-    return Duration.ZERO;
-  }
-
-  /**
-   * Interface for signaling that a {@link OldDoFn} needs to access the window the
-   * element is being processed in, via {@link OldDoFn.ProcessContext#window}.
-   */
-  @Experimental
-  public interface RequiresWindowAccess {}
-
-  public OldDoFn() {
-  }
-
-  /**
-   * Prepares this {@link DoFn} instance for processing bundles.
-   *
-   * <p>{@link #setup()} will be called at most once per {@link DoFn} instance, and before any other
-   * {@link DoFn} method is called.
-   *
-   * <p>By default, does nothing.
-   */
-  public void setup() throws Exception {
-  }
-
-  /**
-   * Prepares this {@code OldDoFn} instance for processing a batch of elements.
-   *
-   * <p>By default, does nothing.
-   */
-  public void startBundle(Context c) throws Exception {
-  }
-
-  /**
-   * Processes one input element.
-   *
-   * <p>The current element of the input {@code PCollection} is returned by
-   * {@link ProcessContext#element() c.element()}. It should be considered immutable. The Beam
-   * runner will not mutate the element, so it is safe to cache, etc. The element should not be
-   * mutated by any of the {@link OldDoFn} methods, because it may be cached elsewhere, retained by
-   * the Beam runner, or used in other unspecified ways.
-   *
-   * <p>A value is added to the main output {@code PCollection} by {@link ProcessContext#output}.
-   * Once passed to {@code output} the element should be considered immutable and not be modified in
-   * any way. It may be cached elsewhere, retained by the Beam runner, or used in other
-   * unspecified ways.
-   *
-   * @see ProcessContext
-   */
-  public abstract void processElement(ProcessContext c) throws Exception;
-
-  /**
-   * Finishes processing this batch of elements.
-   *
-   * <p>By default, does nothing.
-   */
-  public void finishBundle(Context c) throws Exception {
-  }
-
-  /**
-   * Cleans up this {@link DoFn}.
-   *
-   * <p>{@link #teardown()} will be called before the {@link PipelineRunner} discards a {@link DoFn}
-   * instance, including due to another {@link DoFn} method throwing an {@link Exception}. No other
-   * {@link DoFn} methods will be called after a call to {@link #teardown()}.
-   *
-   * <p>By default, does nothing.
-   */
-  public void teardown() throws Exception {
-  }
-
-  /////////////////////////////////////////////////////////////////////////////
-
-  /**
-   * {@inheritDoc}
-   *
-   * <p>By default, does not register any display data. Implementors may override this method
-   * to provide their own display data.
-   */
-  @Override
-  public void populateDisplayData(DisplayData.Builder builder) {
-  }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
deleted file mode 100644
index 2a0b688142b6..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleOldDoFnRunner.java
+++ /dev/null
@@ -1,499 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
-import org.apache.beam.runners.core.OldDoFn.RequiresWindowAccess;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.state.TimeDomain;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.util.UserCodeException;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.joda.time.Instant;
-import org.joda.time.format.PeriodFormat;
-
-/**
- * Runs a {@link OldDoFn} by constructing the appropriate contexts and passing them in.
- *
- * @param <InputT> the type of the {@link OldDoFn} (main) input elements
- * @param <OutputT> the type of the {@link OldDoFn} (main) output elements
- */
-class SimpleOldDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
-
-  /** The {@link OldDoFn} being run. */
-  private final OldDoFn<InputT, OutputT> fn;
-  /** The context used for running the {@link OldDoFn}. */
-  private final DoFnContext<InputT, OutputT> context;
-
-  public SimpleOldDoFnRunner(
-      PipelineOptions options,
-      OldDoFn<InputT, OutputT> fn,
-      SideInputReader sideInputReader,
-      OutputManager outputManager,
-      TupleTag<OutputT> mainOutputTag,
-      List<TupleTag<?>> additionalOutputTags,
-      StepContext stepContext,
-      WindowingStrategy<?, ?> windowingStrategy) {
-    this.fn = fn;
-    this.context = new DoFnContext<>(
-        options,
-        fn,
-        sideInputReader,
-        outputManager,
-        mainOutputTag,
-        additionalOutputTags,
-        stepContext,
-        windowingStrategy == null ? null : windowingStrategy.getWindowFn());
-  }
-
-  @Override
-  public void startBundle() {
-    // This can contain user code. Wrap it in case it throws an exception.
-    try {
-      fn.startBundle(context);
-    } catch (Throwable t) {
-      // Exception in user code.
-      throw wrapUserCodeException(t);
-    }
-  }
-
-  @Override
-  public void processElement(WindowedValue<InputT> elem) {
-    if (elem.getWindows().size() <= 1
-        || (!RequiresWindowAccess.class.isAssignableFrom(fn.getClass())
-            && context.sideInputReader.isEmpty())) {
-      invokeProcessElement(elem);
-    } else {
-      // We could modify the windowed value (and the processContext) to
-      // avoid repeated allocations, but this is more straightforward.
-      for (WindowedValue<InputT> windowedValue : elem.explodeWindows()) {
-        invokeProcessElement(windowedValue);
-      }
-    }
-  }
-
-  @Override
-  public void onTimer(String timerId, BoundedWindow window, Instant timestamp,
-      TimeDomain timeDomain) {
-    throw new UnsupportedOperationException(
-        String.format("Timers are not supported by %s", OldDoFn.class.getSimpleName()));
-  }
-
-  private void invokeProcessElement(WindowedValue<InputT> elem) {
-    final OldDoFn<InputT, OutputT>.ProcessContext processContext = createProcessContext(elem);
-    // This can contain user code. Wrap it in case it throws an exception.
-    try {
-      fn.processElement(processContext);
-    } catch (Exception ex) {
-      throw wrapUserCodeException(ex);
-    }
-  }
-
-  @Override
-  public void finishBundle() {
-    // This can contain user code. Wrap it in case it throws an exception.
-    try {
-      fn.finishBundle(context);
-    } catch (Throwable t) {
-      // Exception in user code.
-      throw wrapUserCodeException(t);
-    }
-  }
-
-  /**
-   * Returns a new {@link OldDoFn.ProcessContext} for the given element.
-   */
-  private OldDoFn<InputT, OutputT>.ProcessContext createProcessContext(
-      WindowedValue<InputT> elem) {
-    return new DoFnProcessContext<InputT, OutputT>(fn, context, elem);
-  }
-
-  private RuntimeException wrapUserCodeException(Throwable t) {
-    throw UserCodeException.wrapIf(!isSystemDoFn(), t);
-  }
-
-  private boolean isSystemDoFn() {
-    return fn.getClass().isAnnotationPresent(SystemDoFnInternal.class);
-  }
-
-  /**
-   * A concrete implementation of {@code OldDoFn.Context} used for running a {@link OldDoFn}.
-   *
-   * @param <InputT> the type of the {@link OldDoFn} (main) input elements
-   * @param <OutputT> the type of the {@link OldDoFn} (main) output elements
-   */
-  private static class DoFnContext<InputT, OutputT> extends OldDoFn<InputT, OutputT>.Context {
-    private static final int MAX_SIDE_OUTPUTS = 1000;
-
-    final PipelineOptions options;
-    final OldDoFn<InputT, OutputT> fn;
-    final SideInputReader sideInputReader;
-    final OutputManager outputManager;
-    final TupleTag<OutputT> mainOutputTag;
-    final StepContext stepContext;
-    final WindowFn<?, ?> windowFn;
-
-    /**
-     * The set of known output tags, some of which may be undeclared, so we can throw an
-     * exception when it exceeds {@link #MAX_SIDE_OUTPUTS}.
-     */
-    private Set<TupleTag<?>> outputTags;
-
-    public DoFnContext(PipelineOptions options,
-                       OldDoFn<InputT, OutputT> fn,
-                       SideInputReader sideInputReader,
-                       OutputManager outputManager,
-                       TupleTag<OutputT> mainOutputTag,
-                       List<TupleTag<?>> additionalOutputTags,
-                       StepContext stepContext,
-                       WindowFn<?, ?> windowFn) {
-      fn.super();
-      this.options = options;
-      this.fn = fn;
-      this.sideInputReader = sideInputReader;
-      this.outputManager = outputManager;
-      this.mainOutputTag = mainOutputTag;
-      this.outputTags = Sets.newHashSet();
-
-      outputTags.add(mainOutputTag);
-      for (TupleTag<?> additionalOutputTag : additionalOutputTags) {
-        outputTags.add(additionalOutputTag);
-      }
-
-      this.stepContext = stepContext;
-      this.windowFn = windowFn;
-    }
-
-    //////////////////////////////////////////////////////////////////////////////
-
-    @Override
-    public PipelineOptions getPipelineOptions() {
-      return options;
-    }
-
-    <T, W extends BoundedWindow> WindowedValue<T> makeWindowedValue(
-        T output, Instant timestamp, Collection<W> windows, PaneInfo pane) {
-      final Instant inputTimestamp = timestamp;
-
-      if (timestamp == null) {
-        timestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
-      }
-
-      if (windows == null) {
-        try {
-          // The windowFn can never succeed at accessing the element, so its type does not
-          // matter here
-          @SuppressWarnings("unchecked")
-          WindowFn<Object, W> objectWindowFn = (WindowFn<Object, W>) windowFn;
-          windows = objectWindowFn.assignWindows(objectWindowFn.new AssignContext() {
-            @Override
-            public Object element() {
-              throw new UnsupportedOperationException(
-                  "WindowFn attempted to access input element when none was available");
-            }
-
-            @Override
-            public Instant timestamp() {
-              if (inputTimestamp == null) {
-                throw new UnsupportedOperationException(
-                    "WindowFn attempted to access input timestamp when none was available");
-              }
-              return inputTimestamp;
-            }
-
-            @Override
-            public W window() {
-              throw new UnsupportedOperationException(
-                  "WindowFn attempted to access input windows when none were available");
-            }
-          });
-        } catch (Exception e) {
-          throw UserCodeException.wrap(e);
-        }
-      }
-
-      return WindowedValue.of(output, timestamp, windows, pane);
-    }
-
-    public <T> T sideInput(PCollectionView<T> view, BoundedWindow sideInputWindow) {
-      if (!sideInputReader.contains(view)) {
-        throw new IllegalArgumentException("calling sideInput() with unknown view");
-      }
-      return sideInputReader.get(view, sideInputWindow);
-    }
-
-    void outputWindowedValue(
-        OutputT output,
-        Instant timestamp,
-        Collection<? extends BoundedWindow> windows,
-        PaneInfo pane) {
-      outputWindowedValue(makeWindowedValue(output, timestamp, windows, pane));
-    }
-
-    void outputWindowedValue(WindowedValue<OutputT> windowedElem) {
-      outputManager.output(mainOutputTag, windowedElem);
-      if (stepContext != null) {
-        stepContext.noteOutput(windowedElem);
-      }
-    }
-
-    private <T> void outputWindowedValue(TupleTag<T> tag,
-                                               T output,
-                                               Instant timestamp,
-                                               Collection<? extends BoundedWindow> windows,
-                                               PaneInfo pane) {
-      outputWindowedValue(tag, makeWindowedValue(output, timestamp, windows, pane));
-    }
-
-    private <T> void outputWindowedValue(TupleTag<T> tag, WindowedValue<T> windowedElem) {
-      if (!outputTags.contains(tag)) {
-        // This tag wasn't declared nor was it seen before during this execution.
-        // Thus, this must be a new, undeclared and unconsumed output.
-        // To prevent likely user errors, enforce the limit on the number of side
-        // outputs.
-        if (outputTags.size() >= MAX_SIDE_OUTPUTS) {
-          throw new IllegalArgumentException(
-              "the number of outputs has exceeded a limit of " + MAX_SIDE_OUTPUTS);
-        }
-        outputTags.add(tag);
-      }
-
-      outputManager.output(tag, windowedElem);
-      if (stepContext != null) {
-        stepContext.noteOutput(tag, windowedElem);
-      }
-    }
-
-    // Following implementations of output, outputWithTimestamp, and output
-    // are only accessible in OldDoFn.startBundle and OldDoFn.finishBundle, and will be shadowed by
-    // ProcessContext's versions in OldDoFn.processElement.
-    @Override
-    public void output(OutputT output) {
-      outputWindowedValue(output, null, null, PaneInfo.NO_FIRING);
-    }
-
-    @Override
-    public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      outputWindowedValue(output, timestamp, null, PaneInfo.NO_FIRING);
-    }
-
-    @Override
-    public <T> void output(TupleTag<T> tag, T output) {
-      checkNotNull(tag, "TupleTag passed to output cannot be null");
-      outputWindowedValue(tag, output, null, null, PaneInfo.NO_FIRING);
-    }
-
-    @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      checkNotNull(tag, "TupleTag passed to outputWithTimestamp cannot be null");
-      outputWindowedValue(tag, output, timestamp, null, PaneInfo.NO_FIRING);
-    }
-  }
-
-  /**
-   * A concrete implementation of {@link OldDoFn.ProcessContext} used for running a {@link OldDoFn}
-   * over a single element.
-   *
-   * @param <InputT> the type of the {@link OldDoFn} (main) input elements
-   * @param <OutputT> the type of the {@link OldDoFn} (main) output elements
-   */
-  private static class DoFnProcessContext<InputT, OutputT>
-      extends OldDoFn<InputT, OutputT>.ProcessContext {
-
-
-    final OldDoFn<InputT, OutputT> fn;
-    final DoFnContext<InputT, OutputT> context;
-    final WindowedValue<InputT> windowedValue;
-
-    public DoFnProcessContext(OldDoFn<InputT, OutputT> fn,
-                              DoFnContext<InputT, OutputT> context,
-                              WindowedValue<InputT> windowedValue) {
-      fn.super();
-      this.fn = fn;
-      this.context = context;
-      this.windowedValue = windowedValue;
-    }
-
-    @Override
-    public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
-    }
-
-    @Override
-    public InputT element() {
-      return windowedValue.getValue();
-    }
-
-    @Override
-    public <T> T sideInput(PCollectionView<T> view) {
-      checkNotNull(view, "View passed to sideInput cannot be null");
-      Iterator<? extends BoundedWindow> windowIter = windows().iterator();
-      BoundedWindow window;
-      if (!windowIter.hasNext()) {
-        if (context.windowFn instanceof GlobalWindows) {
-          // TODO: Remove this once GroupByKeyOnly no longer outputs elements
-          // without windows
-          window = GlobalWindow.INSTANCE;
-        } else {
-          throw new IllegalStateException(
-              "sideInput called when main input element is not in any windows");
-        }
-      } else {
-        window = windowIter.next();
-        if (windowIter.hasNext()) {
-          throw new IllegalStateException(
-              "sideInput called when main input element is in multiple windows");
-        }
-      }
-      return context.sideInput(
-          view, view.getWindowMappingFn().getSideInputWindow(window));
-    }
-
-    @Override
-    public BoundedWindow window() {
-      if (!(fn instanceof OldDoFn.RequiresWindowAccess)) {
-        throw new UnsupportedOperationException(
-            "window() is only available in the context of a OldDoFn marked as"
-                + "RequiresWindowAccess.");
-      }
-      return Iterables.getOnlyElement(windows());
-    }
-
-    @Override
-    public PaneInfo pane() {
-      return windowedValue.getPane();
-    }
-
-    @Override
-    public void output(OutputT output) {
-      context.outputWindowedValue(windowedValue.withValue(output));
-    }
-
-    @Override
-    public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      checkTimestamp(timestamp);
-      context.outputWindowedValue(output, timestamp,
-          windowedValue.getWindows(), windowedValue.getPane());
-    }
-
-    void outputWindowedValue(
-        OutputT output,
-        Instant timestamp,
-        Collection<? extends BoundedWindow> windows,
-        PaneInfo pane) {
-      context.outputWindowedValue(output, timestamp, windows, pane);
-    }
-
-    @Override
-    public <T> void output(TupleTag<T> tag, T output) {
-      checkNotNull(tag, "Tag passed to output cannot be null");
-      context.outputWindowedValue(tag, windowedValue.withValue(output));
-    }
-
-    @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      checkNotNull(tag, "Tag passed to outputWithTimestamp cannot be null");
-      checkTimestamp(timestamp);
-      context.outputWindowedValue(
-          tag, output, timestamp, windowedValue.getWindows(), windowedValue.getPane());
-    }
-
-    @Override
-    public Instant timestamp() {
-      return windowedValue.getTimestamp();
-    }
-
-    public Collection<? extends BoundedWindow> windows() {
-      return windowedValue.getWindows();
-    }
-
-    private void checkTimestamp(Instant timestamp) {
-      if (timestamp.isBefore(windowedValue.getTimestamp().minus(fn.getAllowedTimestampSkew()))) {
-        throw new IllegalArgumentException(String.format(
-            "Cannot output with timestamp %s. Output timestamps must be no earlier than the "
-            + "timestamp of the current input (%s) minus the allowed skew (%s). See the "
-            + "OldDoFn#getAllowedTimestampSkew() Javadoc for details on changing the allowed skew.",
-            timestamp, windowedValue.getTimestamp(),
-            PeriodFormat.getDefault().print(fn.getAllowedTimestampSkew().toPeriod())));
-      }
-    }
-
-    @Override
-    public WindowingInternals<InputT, OutputT> windowingInternals() {
-      return new WindowingInternals<InputT, OutputT>() {
-        @Override
-        public void outputWindowedValue(OutputT output, Instant timestamp,
-            Collection<? extends BoundedWindow> windows, PaneInfo pane) {
-          context.outputWindowedValue(output, timestamp, windows, pane);
-        }
-
-        @Override
-        public <AdditionalOutputT> void outputWindowedValue(
-            TupleTag<AdditionalOutputT> tag,
-            AdditionalOutputT output,
-            Instant timestamp,
-            Collection<? extends BoundedWindow> windows,
-            PaneInfo pane) {
-          context.outputWindowedValue(tag, output, timestamp, windows, pane);
-        }
-
-        @Override
-        public Collection<? extends BoundedWindow> windows() {
-          return windowedValue.getWindows();
-        }
-
-        @Override
-        public PaneInfo pane() {
-          return windowedValue.getPane();
-        }
-
-        @Override
-        public TimerInternals timerInternals() {
-          return context.stepContext.timerInternals();
-        }
-
-        @Override
-        public StateInternals stateInternals() {
-          return context.stepContext.stateInternals();
-        }
-
-        @Override
-        public <T> T sideInput(PCollectionView<T> view, BoundedWindow sideInputWindow) {
-          return context.sideInput(view, sideInputWindow);
-        }
-      };
-    }
-  }
-}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java
deleted file mode 100644
index 4a5844562980..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/WindowingInternalsAdapters.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.util.Collection;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.joda.time.Instant;
-
-/**
- * Adapters from {@link WindowingInternals} to {@link SideInputReader} and {@link
- * OutputWindowedValue}.
- */
-public class WindowingInternalsAdapters {
-  static SideInputReader sideInputReader(final WindowingInternals<?, ?> windowingInternals) {
-    return new SideInputReader() {
-      @Override
-      public <T> T get(PCollectionView<T> view, BoundedWindow sideInputWindow) {
-        return windowingInternals.sideInput(view, sideInputWindow);
-      }
-
-      @Override
-      public <T> boolean contains(PCollectionView<T> view) {
-        throw new UnsupportedOperationException();
-      }
-
-      @Override
-      public boolean isEmpty() {
-        throw new UnsupportedOperationException();
-      }
-    };
-  }
-
-  public static <OutputT> OutputWindowedValue<OutputT> outputWindowedValue(
-      final WindowingInternals<?, OutputT> windowingInternals) {
-    return new OutputWindowedValue<OutputT>() {
-      @Override
-      public void outputWindowedValue(
-          OutputT output,
-          Instant timestamp,
-          Collection<? extends BoundedWindow> windows,
-          PaneInfo pane) {
-        windowingInternals.outputWindowedValue(output, timestamp, windows, pane);
-      }
-
-      @Override
-      public <AdditionalOutputT> void outputWindowedValue(
-          TupleTag<AdditionalOutputT> tag,
-          AdditionalOutputT output,
-          Instant timestamp,
-          Collection<? extends BoundedWindow> windows,
-          PaneInfo pane) {
-        windowingInternals.outputWindowedValue(tag, output, timestamp, windows, pane);
-      }
-    };
-  }
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java
deleted file mode 100644
index a265eade4ffd..000000000000
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowViaOutputBufferDoFnTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import org.apache.beam.runners.core.GroupAlsoByWindowsProperties.GroupAlsoByWindowsDoFnFactory;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Unit tests for {@link GroupAlsoByWindowViaOutputBufferDoFn}.
- */
-@RunWith(JUnit4.class)
-public class GroupAlsoByWindowViaOutputBufferDoFnTest {
-
-  private class BufferingGABWViaOutputBufferDoFnFactory<K, InputT>
-  implements GroupAlsoByWindowsDoFnFactory<K, InputT, Iterable<InputT>> {
-
-    private final Coder<InputT> inputCoder;
-
-    public BufferingGABWViaOutputBufferDoFnFactory(Coder<InputT> inputCoder) {
-      this.inputCoder = inputCoder;
-    }
-
-    @Override
-    public <W extends BoundedWindow>
-    GroupAlsoByWindowsDoFn<K, InputT, Iterable<InputT>, W> forStrategy(
-            WindowingStrategy<?, W> windowingStrategy,
-            StateInternalsFactory<K> stateInternalsFactory) {
-      return new GroupAlsoByWindowViaOutputBufferDoFn<K, InputT, Iterable<InputT>, W>(
-          windowingStrategy,
-          stateInternalsFactory,
-          SystemReduceFn.<K, InputT, W>buffering(inputCoder));
-    }
-  }
-
-  @Test
-  public void testEmptyInputEmptyOutput() throws Exception {
-    GroupAlsoByWindowsProperties.emptyInputEmptyOutput(
-        new BufferingGABWViaOutputBufferDoFnFactory<>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsElementsIntoFixedWindows() throws Exception {
-    GroupAlsoByWindowsProperties.groupsElementsIntoFixedWindows(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsElementsIntoSlidingWindows() throws Exception {
-    GroupAlsoByWindowsProperties.groupsElementsIntoSlidingWindowsWithMinTimestamp(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsIntoOverlappingNonmergingWindows() throws Exception {
-    GroupAlsoByWindowsProperties.groupsIntoOverlappingNonmergingWindows(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsIntoSessions() throws Exception {
-    GroupAlsoByWindowsProperties.groupsElementsInMergedSessions(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsElementsIntoFixedWindowsWithEndOfWindowTimestamp() throws Exception {
-    GroupAlsoByWindowsProperties.groupsElementsIntoFixedWindowsWithEndOfWindowTimestamp(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsElementsIntoFixedWindowsWithLatestTimestamp() throws Exception {
-    GroupAlsoByWindowsProperties.groupsElementsIntoFixedWindowsWithLatestTimestamp(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsElementsIntoSessionsWithEndOfWindowTimestamp() throws Exception {
-    GroupAlsoByWindowsProperties.groupsElementsInMergedSessionsWithEndOfWindowTimestamp(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-
-  @Test
-  public void testGroupsElementsIntoSessionsWithLatestTimestamp() throws Exception {
-    GroupAlsoByWindowsProperties.groupsElementsInMergedSessionsWithLatestTimestamp(
-        new BufferingGABWViaOutputBufferDoFnFactory<String, String>(StringUtf8Coder.of()));
-  }
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java
deleted file mode 100644
index 761ffb87024e..000000000000
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/GroupAlsoByWindowsProperties.java
+++ /dev/null
@@ -1,744 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.junit.Assert.assertThat;
-
-import com.google.common.base.Predicate;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.Combine.CombineFn;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.Sessions;
-import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
-import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TimestampedValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-
-/**
- * Properties of {@link GroupAlsoByWindowsDoFn}.
- *
- * <p>Some properties may not hold of some implementations, due to restrictions on the context in
- * which the implementation is applicable. For example, some {@code GroupAlsoByWindows} may not
- * support merging windows.
- */
-public class GroupAlsoByWindowsProperties {
-
-  /**
-   * A factory of {@link GroupAlsoByWindowsDoFn} so that the various properties can provide the
-   * appropriate windowing strategy under test.
-   */
-  public interface GroupAlsoByWindowsDoFnFactory<K, InputT, OutputT> {
-    <W extends BoundedWindow> GroupAlsoByWindowsDoFn<K, InputT, OutputT, W> forStrategy(
-        WindowingStrategy<?, W> strategy, StateInternalsFactory<K> stateInternalsFactory);
-  }
-
-  /**
-   * Tests that for empty input and the given {@link WindowingStrategy}, the provided GABW
-   * implementation produces no output.
-   *
-   * <p>The input type is deliberately left as a wildcard, since it is not relevant.
-   */
-  public static <K, InputT, OutputT> void emptyInputEmptyOutput(
-      GroupAlsoByWindowsDoFnFactory<K, InputT, OutputT> gabwFactory) throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(FixedWindows.of(Duration.millis(10)));
-
-    // This key should never actually be used, though it is eagerly passed to the
-    // StateInternalsFactory so must be non-null
-    @SuppressWarnings("unchecked")
-    K fakeKey = (K) "this key should never be used";
-
-    List<WindowedValue<KV<K, OutputT>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            fakeKey,
-            Collections.<WindowedValue<InputT>>emptyList());
-
-    assertThat(result, hasSize(0));
-  }
-
-  /**
-   * Tests that for a simple sequence of elements on the same key, the given GABW implementation
-   * correctly groups them according to fixed windows.
-   */
-  public static void groupsElementsIntoFixedWindows(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(FixedWindows.of(Duration.millis(10)));
-
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "key",
-            WindowedValue.of(
-                "v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, window(0, 10));
-    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
-    assertThat(item0.getTimestamp(), equalTo(window(0, 10).maxTimestamp()));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, window(10, 20));
-    assertThat(item1.getValue().getValue(), contains("v3"));
-    assertThat(item1.getTimestamp(), equalTo(window(10, 20).maxTimestamp()));
-  }
-
-  /**
-   * Tests that for a simple sequence of elements on the same key, the given GABW implementation
-   * correctly groups them into sliding windows.
-   *
-   * <p>In the input here, each element occurs in multiple windows.
-   */
-  public static void groupsElementsIntoSlidingWindowsWithMinTimestamp(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(SlidingWindows.of(Duration.millis(20)).every(Duration.millis(10)))
-            .withTimestampCombiner(TimestampCombiner.EARLIEST);
-
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "key",
-            WindowedValue.of(
-                "v1",
-                new Instant(5),
-                Arrays.asList(window(-10, 10), window(0, 20)),
-                PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v2",
-                new Instant(15),
-                Arrays.asList(window(0, 20), window(10, 30)),
-                PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(3));
-
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, window(-10, 10));
-    assertThat(item0.getValue().getValue(), contains("v1"));
-    assertThat(item0.getTimestamp(), equalTo(new Instant(5)));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, window(0, 20));
-    assertThat(item1.getValue().getValue(), containsInAnyOrder("v1", "v2"));
-    // Timestamp adjusted by WindowFn to exceed the end of the prior sliding window
-    assertThat(item1.getTimestamp(), equalTo(new Instant(10)));
-
-    TimestampedValue<KV<String, Iterable<String>>> item2 =
-        getOnlyElementInWindow(result, window(10, 30));
-    assertThat(item2.getValue().getValue(), contains("v2"));
-    // Timestamp adjusted by WindowFn to exceed the end of the prior sliding window
-    assertThat(item2.getTimestamp(), equalTo(new Instant(20)));
-  }
-
-  /**
-   * Tests that for a simple sequence of elements on the same key, the given GABW implementation
-   * correctly groups and combines them according to sliding windows.
-   *
-   * <p>In the input here, each element occurs in multiple windows.
-   */
-  public static void combinesElementsInSlidingWindows(
-      GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory,
-      CombineFn<Long, ?, Long> combineFn)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(SlidingWindows.of(Duration.millis(20)).every(Duration.millis(10)))
-            .withTimestampCombiner(TimestampCombiner.EARLIEST);
-
-    List<WindowedValue<KV<String, Long>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "k",
-            WindowedValue.of(
-                1L,
-                new Instant(5),
-                Arrays.asList(window(-10, 10), window(0, 20)),
-                PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                2L,
-                new Instant(15),
-                Arrays.asList(window(0, 20), window(10, 30)),
-                PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                4L,
-                new Instant(18),
-                Arrays.asList(window(0, 20), window(10, 30)),
-                PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(3));
-
-    TimestampedValue<KV<String, Long>> item0 = getOnlyElementInWindow(result, window(-10, 10));
-    assertThat(item0.getValue().getKey(), equalTo("k"));
-    assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L))));
-    assertThat(item0.getTimestamp(), equalTo(new Instant(5L)));
-
-    TimestampedValue<KV<String, Long>> item1 = getOnlyElementInWindow(result, window(0, 20));
-    assertThat(item1.getValue().getKey(), equalTo("k"));
-    assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L, 4L))));
-    // Timestamp adjusted by WindowFn to exceed the end of the prior sliding window
-    assertThat(item1.getTimestamp(), equalTo(new Instant(10L)));
-
-    TimestampedValue<KV<String, Long>> item2 = getOnlyElementInWindow(result, window(10, 30));
-    assertThat(item2.getValue().getKey(), equalTo("k"));
-    assertThat(item2.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(2L, 4L))));
-    // Timestamp adjusted by WindowFn to exceed the end of the prior sliding window
-    assertThat(item2.getTimestamp(), equalTo(new Instant(20L)));
-  }
-
-  /**
-   * Tests that the given GABW implementation correctly groups elements that fall into overlapping
-   * windows that are not merged.
-   */
-  public static void groupsIntoOverlappingNonmergingWindows(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(FixedWindows.of(Duration.millis(10)));
-
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "key",
-            WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING),
-            WindowedValue.of("v2", new Instant(4), Arrays.asList(window(1, 5)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v3", new Instant(4), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, window(0, 5));
-    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v3"));
-    assertThat(item0.getTimestamp(), equalTo(window(1, 5).maxTimestamp()));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, window(1, 5));
-    assertThat(item1.getValue().getValue(), contains("v2"));
-    assertThat(item1.getTimestamp(), equalTo(window(0, 5).maxTimestamp()));
-  }
-
-  /** Tests that the given GABW implementation correctly groups elements into merged sessions. */
-  public static void groupsElementsInMergedSessions(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(10)));
-
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "key",
-            WindowedValue.of(
-                "v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, window(0, 15));
-    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
-    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, window(15, 25));
-    assertThat(item1.getValue().getValue(), contains("v3"));
-    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
-  }
-
-  /**
-   * Tests that the given {@link GroupAlsoByWindowsDoFn} implementation combines elements per
-   * session window correctly according to the provided {@link CombineFn}.
-   */
-  public static void combinesElementsPerSession(
-      GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory,
-      CombineFn<Long, ?, Long> combineFn)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(10)));
-
-    List<WindowedValue<KV<String, Long>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "k",
-            WindowedValue.of(1L, new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(2L, new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                4L, new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    TimestampedValue<KV<String, Long>> item0 = getOnlyElementInWindow(result, window(0, 15));
-    assertThat(item0.getValue().getKey(), equalTo("k"));
-    assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L))));
-    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));
-
-    TimestampedValue<KV<String, Long>> item1 = getOnlyElementInWindow(result, window(15, 25));
-    assertThat(item1.getValue().getKey(), equalTo("k"));
-    assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(4L))));
-    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
-  }
-
-  /**
-   * Tests that for a simple sequence of elements on the same key, the given GABW implementation
-   * correctly groups them according to fixed windows and also sets the output timestamp according
-   * to the policy {@link TimestampCombiner#END_OF_WINDOW}.
-   */
-  public static void groupsElementsIntoFixedWindowsWithEndOfWindowTimestamp(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
-            .withTimestampCombiner(TimestampCombiner.END_OF_WINDOW);
-
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "key",
-            WindowedValue.of(
-                "v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, window(0, 10));
-    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
-    assertThat(item0.getTimestamp(), equalTo(window(0, 10).maxTimestamp()));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, window(10, 20));
-    assertThat(item1.getValue().getValue(), contains("v3"));
-    assertThat(item1.getTimestamp(), equalTo(window(10, 20).maxTimestamp()));
-  }
-
-  /**
-   * Tests that for a simple sequence of elements on the same key, the given GABW implementation
-   * correctly groups them according to fixed windows and also sets the output timestamp according
-   * to the policy {@link TimestampCombiner#LATEST}.
-   */
-  public static void groupsElementsIntoFixedWindowsWithLatestTimestamp(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
-            .withTimestampCombiner(TimestampCombiner.LATEST);
-
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "k",
-            WindowedValue.of(
-                "v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, window(0, 10));
-    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
-    assertThat(item0.getTimestamp(), equalTo(new Instant(2)));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, window(10, 20));
-    assertThat(item1.getValue().getValue(), contains("v3"));
-    assertThat(item1.getTimestamp(), equalTo(new Instant(13)));
-  }
-
-  /**
-   * Tests that the given GABW implementation correctly groups elements into merged sessions with
-   * output timestamps at the end of the merged window.
-   */
-  public static void groupsElementsInMergedSessionsWithEndOfWindowTimestamp(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(10)))
-            .withTimestampCombiner(TimestampCombiner.END_OF_WINDOW);
-
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "k",
-            WindowedValue.of(
-                "v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, window(0, 15));
-    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
-    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, window(15, 25));
-    assertThat(item1.getValue().getValue(), contains("v3"));
-    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
-  }
-
-  /**
-   * Tests that the given GABW implementation correctly groups elements into merged sessions with
-   * output timestamps at the end of the merged window.
-   */
-  public static void groupsElementsInMergedSessionsWithLatestTimestamp(
-      GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(10)))
-            .withTimestampCombiner(TimestampCombiner.LATEST);
-
-    BoundedWindow unmergedWindow = window(15, 25);
-    List<WindowedValue<KV<String, Iterable<String>>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "k",
-            WindowedValue.of(
-                "v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
-            WindowedValue.of(
-                "v3", new Instant(15), Arrays.asList(unmergedWindow), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    BoundedWindow mergedWindow = window(0, 15);
-    TimestampedValue<KV<String, Iterable<String>>> item0 =
-        getOnlyElementInWindow(result, mergedWindow);
-    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
-    assertThat(item0.getTimestamp(), equalTo(new Instant(5)));
-
-    TimestampedValue<KV<String, Iterable<String>>> item1 =
-        getOnlyElementInWindow(result, unmergedWindow);
-    assertThat(item1.getValue().getValue(), contains("v3"));
-    assertThat(item1.getTimestamp(), equalTo(new Instant(15)));
-  }
-
-  /**
-   * Tests that the given {@link GroupAlsoByWindowsDoFn} implementation combines elements per
-   * session window correctly according to the provided {@link CombineFn}.
-   */
-  public static void combinesElementsPerSessionWithEndOfWindowTimestamp(
-      GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory,
-      CombineFn<Long, ?, Long> combineFn)
-      throws Exception {
-
-    WindowingStrategy<?, IntervalWindow> windowingStrategy =
-        WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(10)))
-            .withTimestampCombiner(TimestampCombiner.END_OF_WINDOW);
-
-    BoundedWindow secondWindow = window(15, 25);
-    List<WindowedValue<KV<String, Long>>> result =
-        runGABW(
-            gabwFactory,
-            windowingStrategy,
-            "k",
-            WindowedValue.of(1L, new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
-            WindowedValue.of(2L, new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
-            WindowedValue.of(4L, new Instant(15), Arrays.asList(secondWindow), PaneInfo.NO_FIRING));
-
-    assertThat(result, hasSize(2));
-
-    BoundedWindow firstResultWindow = window(0, 15);
-    TimestampedValue<KV<String, Long>> item0 = getOnlyElementInWindow(result, firstResultWindow);
-    assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L))));
-    assertThat(item0.getTimestamp(), equalTo(firstResultWindow.maxTimestamp()));
-
-    TimestampedValue<KV<String, Long>> item1 = getOnlyElementInWindow(result, secondWindow);
-    assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(4L))));
-    assertThat(item1.getTimestamp(), equalTo(secondWindow.maxTimestamp()));
-  }
-
-  @SafeVarargs
-  private static <K, InputT, OutputT, W extends BoundedWindow>
-      List<WindowedValue<KV<K, OutputT>>> runGABW(
-          GroupAlsoByWindowsDoFnFactory<K, InputT, OutputT> gabwFactory,
-          WindowingStrategy<?, W> windowingStrategy,
-          K key,
-          WindowedValue<InputT>... values)
-          throws Exception {
-    return runGABW(gabwFactory, windowingStrategy, key, Arrays.asList(values));
-  }
-
-  private static <K, InputT, OutputT, W extends BoundedWindow>
-      List<WindowedValue<KV<K, OutputT>>> runGABW(
-          GroupAlsoByWindowsDoFnFactory<K, InputT, OutputT> gabwFactory,
-          WindowingStrategy<?, W> windowingStrategy,
-          K key,
-          Collection<WindowedValue<InputT>> values)
-          throws Exception {
-
-    final StateInternalsFactory<K> stateInternalsCache = new CachingStateInternalsFactory<K>();
-
-    List<WindowedValue<KV<K, OutputT>>> output =
-        processElement(
-            gabwFactory.forStrategy(windowingStrategy, stateInternalsCache),
-            KV.<K, Iterable<WindowedValue<InputT>>>of(key, values));
-
-    // Sanity check for corruption
-    for (WindowedValue<KV<K, OutputT>> value : output) {
-      assertThat(value.getValue().getKey(), equalTo(key));
-    }
-
-    return output;
-  }
-
-  private static BoundedWindow window(long start, long end) {
-    return new IntervalWindow(new Instant(start), new Instant(end));
-  }
-
-  private static final class CachingStateInternalsFactory<K> implements StateInternalsFactory<K> {
-    private final LoadingCache<K, StateInternals> stateInternalsCache;
-
-    private CachingStateInternalsFactory() {
-      this.stateInternalsCache = CacheBuilder.newBuilder().build(new StateInternalsLoader<K>());
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public StateInternals stateInternalsForKey(K key) {
-      try {
-        return stateInternalsCache.get(key);
-      } catch (Exception exc) {
-        throw new RuntimeException(exc);
-      }
-    }
-  }
-
-  private static class StateInternalsLoader<K> extends CacheLoader<K, StateInternals> {
-    @Override
-    public StateInternals load(K key) throws Exception {
-      return InMemoryStateInternals.forKey(key);
-    }
-  }
-
-  private static <K, InputT, OutputT, W extends BoundedWindow>
-      List<WindowedValue<KV<K, OutputT>>> processElement(
-          GroupAlsoByWindowsDoFn<K, InputT, OutputT, W> fn,
-          KV<K, Iterable<WindowedValue<InputT>>> element)
-          throws Exception {
-    TestProcessContext<K, InputT, OutputT, W> c = new TestProcessContext<>(fn, element);
-    fn.processElement(c);
-    return c.getOutput();
-  }
-
-  private static <K, OutputT> TimestampedValue<KV<K, OutputT>> getOnlyElementInWindow(
-      List<WindowedValue<KV<K, OutputT>>> output, final BoundedWindow window) {
-    WindowedValue<KV<K, OutputT>> res =
-        Iterables.getOnlyElement(
-            Iterables.filter(
-                output,
-                new Predicate<WindowedValue<KV<K, OutputT>>>() {
-                  @Override
-                  public boolean apply(@Nullable WindowedValue<KV<K, OutputT>> input) {
-                    return input.getWindows().contains(window);
-                  }
-                }));
-    return TimestampedValue.of(res.getValue(), res.getTimestamp());
-  }
-
-  /**
-   * A {@link GroupAlsoByWindowsDoFn.ProcessContext} providing just enough context for a {@link
-   * GroupAlsoByWindowsDoFn} - namely, information about the element and output via {@link
-   * WindowingInternals}, but no side inputs/outputs and no normal output.
-   */
-  private static class TestProcessContext<K, InputT, OutputT, W extends BoundedWindow>
-      extends GroupAlsoByWindowsDoFn<K, InputT, OutputT, W>.ProcessContext {
-    private final PipelineOptions options = PipelineOptionsFactory.create();
-    private final KV<K, Iterable<WindowedValue<InputT>>> element;
-    private final List<WindowedValue<KV<K, OutputT>>> output = new ArrayList<>();
-
-    private TestProcessContext(
-        GroupAlsoByWindowsDoFn<K, InputT, OutputT, W> fn,
-        KV<K, Iterable<WindowedValue<InputT>>> element) {
-      fn.super();
-      this.element = element;
-    }
-
-    @Override
-    public KV<K, Iterable<WindowedValue<InputT>>> element() {
-      return element;
-    }
-
-    @Override
-    public Instant timestamp() {
-      return BoundedWindow.TIMESTAMP_MIN_VALUE;
-    }
-
-    @Override
-    public BoundedWindow window() {
-      return GlobalWindow.INSTANCE;
-    }
-
-    @Override
-    public PaneInfo pane() {
-      return PaneInfo.NO_FIRING;
-    }
-
-    @Override
-    public <T> T sideInput(PCollectionView<T> view) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public WindowingInternals<KV<K, Iterable<WindowedValue<InputT>>>, KV<K, OutputT>>
-        windowingInternals() {
-      return new WindowingInternals<KV<K, Iterable<WindowedValue<InputT>>>, KV<K, OutputT>>() {
-        @Override
-        public void outputWindowedValue(
-            KV<K, OutputT> output,
-            Instant timestamp,
-            Collection<? extends BoundedWindow> windows,
-            PaneInfo pane) {
-          TestProcessContext.this.output.add(WindowedValue.of(output, timestamp, windows, pane));
-        }
-
-        @Override
-        public <AdditionalOutputT> void outputWindowedValue(
-            TupleTag<AdditionalOutputT> tag,
-            AdditionalOutputT output,
-            Instant timestamp,
-            Collection<? extends BoundedWindow> windows,
-            PaneInfo pane) {
-          throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public StateInternals stateInternals() {
-          throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public TimerInternals timerInternals() {
-          throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public Collection<? extends BoundedWindow> windows() {
-          return ImmutableList.of(GlobalWindow.INSTANCE);
-        }
-
-        @Override
-        public PaneInfo pane() {
-          return PaneInfo.NO_FIRING;
-        }
-
-        @Override
-        public <T> T sideInput(PCollectionView<T> view, BoundedWindow sideInputWindow) {
-          throw new UnsupportedOperationException();
-        }
-      };
-    }
-
-    @Override
-    public PipelineOptions getPipelineOptions() {
-      return options;
-    }
-
-    @Override
-    public void output(KV<K, OutputT> output) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void outputWithTimestamp(KV<K, OutputT> output, Instant timestamp) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public <T> void output(TupleTag<T> tag, T output) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      throw new UnsupportedOperationException();
-    }
-
-    public List<WindowedValue<KV<K, OutputT>>> getOutput() {
-      return output;
-    }
-  }
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/NoOpOldDoFn.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/NoOpOldDoFn.java
deleted file mode 100644
index 581c3e0ade0e..000000000000
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/NoOpOldDoFn.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.values.TupleTag;
-import org.joda.time.Instant;
-
-/**
- * A {@link OldDoFn} that does nothing with provided elements. Used for testing
- * methods provided by the {@link OldDoFn} abstract class.
- *
- * @param <InputT> unused.
- * @param <OutputT> unused.
- */
-class NoOpOldDoFn<InputT, OutputT> extends OldDoFn<InputT, OutputT> {
-  @Override
-  public void processElement(OldDoFn<InputT, OutputT>.ProcessContext c) throws Exception {
-  }
-
-  /**
-   * Returns a new NoOp Context.
-   */
-  public OldDoFn<InputT, OutputT>.Context context() {
-    return new NoOpDoFnContext();
-  }
-
-  /**
-   * A {@link OldDoFn.Context} that does nothing and returns exclusively null.
-   */
-  private class NoOpDoFnContext extends OldDoFn<InputT, OutputT>.Context {
-    @Override
-    public PipelineOptions getPipelineOptions() {
-      return null;
-    }
-    @Override
-    public void output(OutputT output) {
-    }
-    @Override
-    public void outputWithTimestamp(OutputT output, Instant timestamp) {
-    }
-    @Override
-    public <T> void output(TupleTag<T> tag, T output) {
-    }
-    @Override
-    public <T> void outputWithTimestamp(TupleTag<T> tag, T output,
-        Instant timestamp) {
-    }
-  }
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/OldDoFnTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/OldDoFnTest.java
deleted file mode 100644
index f608a810bd66..000000000000
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/OldDoFnTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static org.hamcrest.Matchers.empty;
-import static org.junit.Assert.assertThat;
-
-import java.io.Serializable;
-import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for OldDoFn.
- */
-@RunWith(JUnit4.class)
-public class OldDoFnTest implements Serializable {
-
-  @Rule
-  public transient ExpectedException thrown = ExpectedException.none();
-
-  @Test
-  public void testPopulateDisplayDataDefaultBehavior() {
-    OldDoFn<String, String> usesDefault =
-        new OldDoFn<String, String>() {
-          @Override
-          public void processElement(ProcessContext c) throws Exception {}
-        };
-
-    DisplayData data = DisplayData.from(usesDefault);
-    assertThat(data.items(), empty());
-  }
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleOldDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleOldDoFnRunnerTest.java
deleted file mode 100644
index a73ef5eb5ab8..000000000000
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleOldDoFnRunnerTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import static org.hamcrest.Matchers.is;
-import static org.mockito.Mockito.mock;
-
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.runners.core.BaseExecutionContext.StepContext;
-import org.apache.beam.sdk.util.SystemDoFnInternal;
-import org.apache.beam.sdk.util.UserCodeException;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link SimpleOldDoFnRunner} functionality.
- */
-@RunWith(JUnit4.class)
-public class SimpleOldDoFnRunnerTest {
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  @Test
-  public void testExceptionsWrappedAsUserCodeException() {
-    ThrowingDoFn fn = new ThrowingDoFn();
-    DoFnRunner<String, String> runner = createRunner(fn);
-
-    thrown.expect(UserCodeException.class);
-    thrown.expectCause(is(fn.exceptionToThrow));
-
-    runner.processElement(WindowedValue.valueInGlobalWindow("anyValue"));
-  }
-
-  @Test
-  public void testSystemDoFnInternalExceptionsNotWrapped() {
-    ThrowingSystemDoFn fn = new ThrowingSystemDoFn();
-    DoFnRunner<String, String> runner = createRunner(fn);
-
-    thrown.expect(is(fn.exceptionToThrow));
-
-    runner.processElement(WindowedValue.valueInGlobalWindow("anyValue"));
-  }
-
-  private DoFnRunner<String, String> createRunner(OldDoFn<String, String> fn) {
-    // Pass in only necessary parameters for the test
-    List<TupleTag<?>> additionalOutputTags = Arrays.asList();
-    StepContext context = mock(StepContext.class);
-    return new SimpleOldDoFnRunner<>(
-        null, fn, null, null, null, additionalOutputTags, context, null);
-  }
-
-  static class ThrowingDoFn extends OldDoFn<String, String> {
-    final Exception exceptionToThrow =
-        new UnsupportedOperationException("Expected exception");
-
-    @Override
-    public void processElement(ProcessContext c) throws Exception {
-      throw exceptionToThrow;
-    }
-  }
-
-  @SystemDoFnInternal
-  static class ThrowingSystemDoFn extends ThrowingDoFn {
-  }
-}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 1d079d9672d5..84be15dd19cd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -24,7 +24,7 @@
 import com.google.common.collect.Iterables;
 import java.util.ArrayList;
 import java.util.Collection;
-import org.apache.beam.runners.core.GroupAlsoByWindowsDoFn;
+import org.apache.beam.runners.core.GroupAlsoByWindowsAggregators;
 import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly;
 import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupAlsoByWindow;
 import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly;
@@ -143,9 +143,9 @@ public GroupAlsoByWindowEvaluator(
           application.getTransform().getValueCoder(inputBundle.getPCollection().getCoder());
       reduceFn = SystemReduceFn.buffering(valueCoder);
       droppedDueToClosedWindow = Metrics.counter(GroupAlsoByWindowEvaluator.class,
-          GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER);
+          GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER);
       droppedDueToLateness = Metrics.counter(GroupAlsoByWindowEvaluator.class,
-          GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_LATENESS_COUNTER);
+          GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_LATENESS_COUNTER);
     }
 
     @Override
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 1d843f90fb2f..815b6ba5dc02 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -23,7 +23,8 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import org.apache.beam.runners.core.GroupAlsoByWindowsDoFn;
+import org.apache.beam.runners.core.GroupAlsoByWindowsAggregators;
+import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupAlsoByWindow;
 import org.apache.beam.runners.core.LateDataUtils;
 import org.apache.beam.runners.core.OutputWindowedValue;
 import org.apache.beam.runners.core.ReduceFnRunner;
@@ -76,7 +77,7 @@
 import scala.runtime.AbstractFunction1;
 
 /**
- * An implementation of {@link org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetDoFn}
+ * An implementation of {@link GroupAlsoByWindow}
  * logic for grouping by windows and controlling trigger firings and pane accumulation.
  *
  * <p>This implementation is a composite of Spark transformations revolving around state management
@@ -210,10 +211,10 @@ public JavaPairRDD<ByteArray, byte[]> call(
         final MetricsContainerImpl cellProvider = new MetricsContainerImpl("cellProvider");
         final CounterCell droppedDueToClosedWindow = cellProvider.getCounter(
             MetricName.named(SparkGroupAlsoByWindowViaWindowSet.class,
-            GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER));
+            GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER));
         final CounterCell droppedDueToLateness = cellProvider.getCounter(
             MetricName.named(SparkGroupAlsoByWindowViaWindowSet.class,
-                GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_LATENESS_COUNTER));
+                GroupAlsoByWindowsAggregators.DROPPED_DUE_TO_LATENESS_COUNTER));
 
         AbstractIterator<
             Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java
index 18a3dd89990c..3d76a615388a 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java
@@ -22,6 +22,7 @@
 import java.util.Collection;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Window.Assign;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.spark.api.java.function.Function;
@@ -29,7 +30,8 @@
 
 
 /**
- * An implementation of {@link org.apache.beam.runners.core.AssignWindowsDoFn} for the Spark runner.
+ * An implementation of {@link Assign} for the Spark
+ * runner.
  */
 public class SparkAssignWindowFn<T, W extends BoundedWindow>
     implements Function<WindowedValue<T>, WindowedValue<T>> {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
index a70885bec06f..be023356ba54 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
@@ -21,7 +21,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import org.apache.beam.runners.core.GroupAlsoByWindowViaOutputBufferDoFn;
+import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupAlsoByWindow;
 import org.apache.beam.runners.core.InMemoryTimerInternals;
 import org.apache.beam.runners.core.OutputWindowedValue;
 import org.apache.beam.runners.core.ReduceFnRunner;
@@ -45,7 +45,7 @@
 import org.joda.time.Instant;
 
 /**
- * An implementation of {@link GroupAlsoByWindowViaOutputBufferDoFn}
+ * An implementation of {@link GroupAlsoByWindow}
  * for the Spark runner.
  */
 public class SparkGroupAlsoByWindowViaOutputBufferFn<K, InputT, W extends BoundedWindow>

From 24179494a116cfde1c1462dd3125ba558d72f494 Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Fri, 12 May 2017 11:19:38 -0700
Subject: [PATCH 248/387] Fixes an accidentally found bug in SimpleDoFnRunner

---
 .../java/org/apache/beam/runners/core/SimpleDoFnRunner.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 7ca305e51d0a..65384da9f8f7 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -215,7 +215,7 @@ public void finishBundle() {
 
   private DoFnStartBundleContext<InputT, OutputT> createStartBundleContext(
       DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
-    return new DoFnStartBundleContext<>(this.fn, this.context);
+    return new DoFnStartBundleContext<>(fn, context);
   }
 
   private DoFnFinishBundleContext<InputT, OutputT> createFinishBundleContext(

From 383a4ada14b498f0036b7b9235ab46b6caf44acc Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Tue, 16 May 2017 00:00:05 -0400
Subject: [PATCH 249/387] [TRIVIAL] runners-core: delete placeholder

This was used to enable checkin of an empty directory, but the directory
is no longer empty.
---
 runners/core-java/src/test/java/.placeholder | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 runners/core-java/src/test/java/.placeholder

diff --git a/runners/core-java/src/test/java/.placeholder b/runners/core-java/src/test/java/.placeholder
deleted file mode 100644
index e69de29bb2d1..000000000000

From 1a05a6a66fabfaa2968f81a73df6b8a1a6fc1301 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 11 Apr 2017 16:50:47 -0700
Subject: [PATCH 250/387] Translate a Pipeline in SdkComponents

---
 .../core/construction/PTransforms.java        |  17 +--
 .../core/construction/SdkComponents.java      |  50 +++++++++
 .../core/construction/SdkComponentsTest.java  | 100 ++++++++++++++++++
 3 files changed, 160 insertions(+), 7 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
index 6d2c6b6fe053..d25d342bfd65 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
@@ -66,13 +66,16 @@ static RunnerApi.PTransform toProto(
           components.registerPCollection((PCollection<?>) taggedInput.getValue()));
     }
     for (Map.Entry<TupleTag<?>, PValue> taggedOutput : appliedPTransform.getOutputs().entrySet()) {
-      checkArgument(
-          taggedOutput.getValue() instanceof PCollection,
-          "Unexpected output type %s",
-          taggedOutput.getValue().getClass());
-      transformBuilder.putOutputs(
-          toProto(taggedOutput.getKey()),
-          components.registerPCollection((PCollection<?>) taggedOutput.getValue()));
+      // TODO: Remove gating
+      if (taggedOutput.getValue() instanceof PCollection) {
+        checkArgument(
+            taggedOutput.getValue() instanceof PCollection,
+            "Unexpected output type %s",
+            taggedOutput.getValue().getClass());
+        transformBuilder.putOutputs(
+            toProto(taggedOutput.getKey()),
+            components.registerPCollection((PCollection<?>) taggedOutput.getValue()));
+      }
     }
     for (AppliedPTransform<?, ?, ?> subtransform : subtransforms) {
       transformBuilder.addSubtransforms(components.getExistingPTransformId(subtransform));
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index 2de8237c8f36..eb29b9a3ae7a 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -22,16 +22,24 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.base.Equivalence;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
+import com.google.common.collect.ListMultimap;
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
 import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.NameUtils;
 import org.apache.beam.sdk.values.PCollection;
@@ -54,6 +62,48 @@ static SdkComponents create() {
     return new SdkComponents();
   }
 
+  public static RunnerApi.Pipeline translatePipeline(Pipeline p) {
+    final SdkComponents components = create();
+    final Collection<String> rootIds = new HashSet<>();
+    p.traverseTopologically(
+        new PipelineVisitor.Defaults() {
+          private final ListMultimap<Node, AppliedPTransform<?, ?, ?>> children =
+              ArrayListMultimap.create();
+
+          @Override
+          public void leaveCompositeTransform(Node node) {
+            if (node.isRootNode()) {
+              for (AppliedPTransform<?, ?, ?> pipelineRoot : children.get(node)) {
+                rootIds.add(components.getExistingPTransformId(pipelineRoot));
+              }
+            } else {
+              children.put(node.getEnclosingNode(), node.toAppliedPTransform());
+              try {
+                components.registerPTransform(node.toAppliedPTransform(), children.get(node));
+              } catch (IOException e) {
+                throw new RuntimeException(e);
+              }
+            }
+          }
+
+          @Override
+          public void visitPrimitiveTransform(Node node) {
+            children.put(node.getEnclosingNode(), node.toAppliedPTransform());
+            try {
+              components.registerPTransform(
+                  node.toAppliedPTransform(), Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+            } catch (IOException e) {
+              throw new IllegalStateException(e);
+            }
+          }
+        });
+    // TODO: Display Data
+    return RunnerApi.Pipeline.newBuilder()
+        .setComponents(components.toComponents())
+        .addAllRootTransformIds(rootIds)
+        .build();
+  }
+
   private SdkComponents() {
     this.componentsBuilder = RunnerApi.Components.newBuilder();
     this.transformIds = HashBiMap.create();
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
index 82840d670973..7424886d1009 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
@@ -24,25 +24,43 @@
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertThat;
 
+import com.google.common.base.Equivalence;
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SetCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.StructuredCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode;
 import org.hamcrest.Matchers;
+import org.joda.time.Duration;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -59,6 +77,88 @@ public class SdkComponentsTest {
 
   private SdkComponents components = SdkComponents.create();
 
+  @Test
+  public void translatePipeline() {
+    BigEndianLongCoder customCoder = BigEndianLongCoder.of();
+    PCollection<Long> elems = pipeline.apply(GenerateSequence.from(0L).to(207L));
+    PCollection<Long> counted = elems.apply(Count.<Long>globally()).setCoder(customCoder);
+    PCollection<Long> windowed =
+        counted.apply(
+            Window.<Long>into(FixedWindows.of(Duration.standardMinutes(7)))
+                .triggering(
+                    AfterWatermark.pastEndOfWindow()
+                        .withEarlyFirings(AfterPane.elementCountAtLeast(19)))
+                .accumulatingFiredPanes()
+                .withAllowedLateness(Duration.standardMinutes(3L)));
+    final WindowingStrategy<?, ?> windowedStrategy = windowed.getWindowingStrategy();
+    PCollection<KV<String, Long>> keyed = windowed.apply(WithKeys.<String, Long>of("foo"));
+    PCollection<KV<String, Iterable<Long>>> grouped =
+        keyed.apply(GroupByKey.<String, Long>create());
+
+    final RunnerApi.Pipeline pipelineProto = SdkComponents.translatePipeline(pipeline);
+    pipeline.traverseTopologically(
+        new PipelineVisitor() {
+          Set<Node> transforms = new HashSet<>();
+          Set<PCollection<?>> pcollections = new HashSet<>();
+          Set<Equivalence.Wrapper<? extends Coder<?>>> coders = new HashSet<>();
+          Set<WindowingStrategy<?, ?>> windowingStrategies = new HashSet<>();
+
+          @Override
+          public CompositeBehavior enterCompositeTransform(Node node) {
+            return CompositeBehavior.ENTER_TRANSFORM;
+          }
+
+          @Override
+          public void leaveCompositeTransform(Node node) {
+            if (node.isRootNode()) {
+              assertThat(
+                  "Unexpected number of PTransforms",
+                  pipelineProto.getComponents().getTransformsCount(),
+                  equalTo(transforms.size()));
+              assertThat(
+                  "Unexpected number of PCollections",
+                  pipelineProto.getComponents().getPcollectionsCount(),
+                  equalTo(pcollections.size()));
+              assertThat(
+                  "Unexpected number of Coders",
+                  pipelineProto.getComponents().getCodersCount(),
+                  equalTo(coders.size()));
+              assertThat(
+                  "Unexpected number of Windowing Strategies",
+                  pipelineProto.getComponents().getWindowingStrategiesCount(),
+                  equalTo(windowingStrategies.size()));
+            } else {
+              transforms.add(node);
+            }
+          }
+
+          @Override
+          public void visitPrimitiveTransform(Node node) {
+            transforms.add(node);
+          }
+
+          @Override
+          public void visitValue(PValue value, Node producer) {
+            if (value instanceof PCollection) {
+              PCollection pc = (PCollection) value;
+              pcollections.add(pc);
+              addCoders(pc.getCoder());
+              windowingStrategies.add(pc.getWindowingStrategy());
+              addCoders(pc.getWindowingStrategy().getWindowFn().windowCoder());
+            }
+          }
+
+          private void addCoders(Coder<?> coder) {
+            coders.add(Equivalence.<Coder<?>>identity().wrap(coder));
+            if (coder instanceof StructuredCoder) {
+              for (Coder<?> component : ((StructuredCoder <?>) coder).getComponents()) {
+                addCoders(component);
+              }
+            }
+          }
+        });
+  }
+
   @Test
   public void registerCoder() throws IOException {
     Coder<?> coder =

From 6c45ebfb2832caae99f4992920adbb7d19dce005 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 8 Mar 2017 17:02:05 -0800
Subject: [PATCH 251/387] Use stable naming strategy for ByteBuddy invokers

This helps to coalesce related failures across multiple
JVM instances, over time, across machines, etc.
---
 .../reflect/ByteBuddyDoFnInvokerFactory.java  | 10 ++--
 .../ByteBuddyOnTimerInvokerFactory.java       | 20 +++----
 .../reflect/StableInvokerNamingStrategy.java  | 54 +++++++++++++++++++
 .../transforms/reflect/DoFnInvokersTest.java  | 19 +++++++
 .../reflect/OnTimerInvokersTest.java          | 32 +++++++++++
 5 files changed, 118 insertions(+), 17 deletions(-)
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
index 8ae2c65a34ec..5d5887a3c59b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
@@ -29,7 +29,6 @@
 import java.util.Map;
 import javax.annotation.Nullable;
 import net.bytebuddy.ByteBuddy;
-import net.bytebuddy.NamingStrategy;
 import net.bytebuddy.description.field.FieldDescription;
 import net.bytebuddy.description.method.MethodDescription;
 import net.bytebuddy.description.modifier.Visibility;
@@ -282,12 +281,9 @@ public static RestrictionTracker invokeNewTracker(Object restriction) {
             // Create subclasses inside the target class, to have access to
             // private and package-private bits
             .with(
-                new NamingStrategy.SuffixingRandom("auxiliary") {
-                  @Override
-                  public String subclass(TypeDescription.Generic superClass) {
-                    return super.name(clazzDescription);
-                  }
-                })
+                StableInvokerNamingStrategy.forDoFnClass(fnClass)
+                    .withSuffix(DoFnInvoker.class.getSimpleName()))
+
             // class <invoker class> extends DoFnInvokerBase {
             .subclass(DoFnInvokerBase.class, ConstructorStrategy.Default.NO_CONSTRUCTORS)
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java
index 123808c813e0..e03133727978 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java
@@ -21,12 +21,12 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.io.BaseEncoding;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.concurrent.ExecutionException;
 import net.bytebuddy.ByteBuddy;
-import net.bytebuddy.NamingStrategy;
 import net.bytebuddy.description.method.MethodDescription;
 import net.bytebuddy.description.modifier.FieldManifestation;
 import net.bytebuddy.description.modifier.Visibility;
@@ -150,20 +150,20 @@ public Constructor<?> load(String timerId) throws Exception {
 
     final TypeDescription clazzDescription = new TypeDescription.ForLoadedType(fnClass);
 
-    final String className =
-        "auxiliary_OnTimer_" + CharMatcher.JAVA_LETTER_OR_DIGIT.retainFrom(timerId);
+    final String suffix =
+        String.format(
+            "%s$%s$%s",
+            OnTimerInvoker.class.getSimpleName(),
+            CharMatcher.javaLetterOrDigit().retainFrom(timerId),
+            BaseEncoding.base64().omitPadding().encode(timerId.getBytes()));
 
     DynamicType.Builder<?> builder =
         new ByteBuddy()
             // Create subclasses inside the target class, to have access to
             // private and package-private bits
-            .with(
-                new NamingStrategy.SuffixingRandom(className) {
-                  @Override
-                  public String subclass(TypeDescription.Generic superClass) {
-                    return super.name(clazzDescription);
-                  }
-                })
+            .with(StableInvokerNamingStrategy.forDoFnClass(fnClass)
+                .withSuffix(suffix))
+
             // class <invoker class> implements OnTimerInvoker {
             .subclass(OnTimerInvoker.class, ConstructorStrategy.Default.NO_CONSTRUCTORS)
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java
new file mode 100644
index 000000000000..42b9381bc423
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.reflect;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+
+import com.google.auto.value.AutoValue;
+import javax.annotation.Nullable;
+import net.bytebuddy.NamingStrategy;
+import net.bytebuddy.description.type.TypeDescription;
+import org.apache.beam.sdk.transforms.DoFn;
+
+/**
+ * A naming strategy for ByteBuddy invokers ({@link DoFnInvoker} and {@link OnTimerInvoker}) that is
+ * deterministic and readable. This is correct to use only when a class is created at most once.
+ */
+@AutoValue
+abstract class StableInvokerNamingStrategy extends NamingStrategy.AbstractBase {
+
+  public abstract Class<? extends DoFn<?, ?>> getFnClass();
+
+  @Nullable
+  public abstract String getSuffix();
+
+  public static StableInvokerNamingStrategy forDoFnClass(Class<? extends DoFn<?, ?>> fnClass) {
+    return new AutoValue_StableInvokerNamingStrategy(fnClass, null);
+  }
+
+  public StableInvokerNamingStrategy withSuffix(String newSuffix) {
+    return new AutoValue_StableInvokerNamingStrategy(getFnClass(), newSuffix);
+  }
+
+  @Override
+  protected String name(TypeDescription superClass) {
+    return String.format(
+        "%s$%s",
+        getFnClass().getName(), firstNonNull(getSuffix(), superClass.getName().replace(".", "_")));
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index a8cd35ed7066..3edb19478b3f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -694,4 +694,23 @@ public void onMyTimer(IntervalWindow w) {
     invoker.invokeOnTimer(timerId, mockArgumentProvider);
     assertThat(fn.window, equalTo(testWindow));
   }
+
+  static class StableNameTestDoFn extends DoFn<Void, Void> {
+    @ProcessElement
+    public void process() {}
+  };
+
+  /**
+   * This is a change-detector test that the generated name is stable across runs.
+   */
+  @Test
+  public void testStableName() {
+    DoFnInvoker<Void, Void> invoker = DoFnInvokers.invokerFor(new StableNameTestDoFn());
+    assertThat(
+        invoker.getClass().getName(),
+        equalTo(
+            String.format(
+                "%s$%s", StableNameTestDoFn.class.getName(), DoFnInvoker.class.getSimpleName())));
+  }
+
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java
index d31795216943..0cc67c63b662 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/OnTimerInvokersTest.java
@@ -105,4 +105,36 @@ public void onMyTimer(BoundedWindow window) {
       this.window = window;
     }
   }
+
+  static class StableNameTestDoFn extends DoFn<Void, Void> {
+    private static final String TIMER_ID = "timer-id.with specialChars{}";
+
+    @TimerId(TIMER_ID)
+    private final TimerSpec myTimer = TimerSpecs.timer(TimeDomain.PROCESSING_TIME);
+
+    @ProcessElement
+    public void process() {}
+
+    @OnTimer(TIMER_ID)
+    public void onMyTimer() {}
+  };
+
+  /**
+   * This is a change-detector test that the generated name is stable across runs.
+   */
+  @Test
+  public void testStableName() {
+    OnTimerInvoker<Void, Void> invoker =
+        OnTimerInvokers.forTimer(new StableNameTestDoFn(), StableNameTestDoFn.TIMER_ID);
+
+    assertThat(
+        invoker.getClass().getName(),
+        equalTo(
+            String.format(
+                "%s$%s$%s$%s",
+                StableNameTestDoFn.class.getName(),
+                OnTimerInvoker.class.getSimpleName(),
+                "timeridwithspecialChars" /* alphanum only; human readable but not unique */,
+                "dGltZXItaWQud2l0aCBzcGVjaWFsQ2hhcnN7fQ" /* base64 encoding of UTF-8 timerId */)));
+  }
 }

From 9b5167483c89423a0789658a33e51b3f5b6e8e16 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 18 May 2017 08:32:29 -0700
Subject: [PATCH 252/387] Restore non-recursive globs in GcsUtil

---
 .../java/org/apache/beam/sdk/util/GcsUtil.java   |  8 +++++++-
 .../org/apache/beam/sdk/util/GcsUtilTest.java    | 16 +++++++++++-----
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
index 94b733a7f8dc..8d1fe74ad270 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
@@ -176,7 +176,13 @@ public static String wildcardToRegexp(String globExp) {
       char c = src[i++];
       switch (c) {
         case '*':
-          dst.append(".*");
+          // One char lookahead for **
+          if (i < src.length && src[i] == '*') {
+            dst.append(".*");
+            ++i;
+          } else {
+            dst.append("[^/]*");
+          }
           break;
         case '?':
           dst.append("[^/]");
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
index 532645047d5a..2164e75c9cfe 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
@@ -93,11 +93,17 @@ public class GcsUtilTest {
   @Test
   public void testGlobTranslation() {
     assertEquals("foo", GcsUtil.wildcardToRegexp("foo"));
-    assertEquals("fo.*o", GcsUtil.wildcardToRegexp("fo*o"));
-    assertEquals("f.*o\\.[^/]", GcsUtil.wildcardToRegexp("f*o.?"));
-    assertEquals("foo-[0-9].*", GcsUtil.wildcardToRegexp("foo-[0-9]*"));
-    assertEquals(".*.*foo", GcsUtil.wildcardToRegexp("**/*foo"));
-    assertEquals(".*.*foo", GcsUtil.wildcardToRegexp("**foo"));
+    assertEquals("fo[^/]*o", GcsUtil.wildcardToRegexp("fo*o"));
+    assertEquals("f[^/]*o\\.[^/]", GcsUtil.wildcardToRegexp("f*o.?"));
+    assertEquals("foo-[0-9][^/]*", GcsUtil.wildcardToRegexp("foo-[0-9]*"));
+    assertEquals("foo-[0-9].*", GcsUtil.wildcardToRegexp("foo-[0-9]**"));
+    assertEquals(".*foo", GcsUtil.wildcardToRegexp("**/*foo"));
+    assertEquals(".*foo", GcsUtil.wildcardToRegexp("**foo"));
+    assertEquals("foo/[^/]*", GcsUtil.wildcardToRegexp("foo/*"));
+    assertEquals("foo[^/]*", GcsUtil.wildcardToRegexp("foo*"));
+    assertEquals("foo/[^/]*/[^/]*/[^/]*", GcsUtil.wildcardToRegexp("foo/*/*/*"));
+    assertEquals("foo/[^/]*/.*", GcsUtil.wildcardToRegexp("foo/*/**"));
+    assertEquals("foo.*baz", GcsUtil.wildcardToRegexp("foo**baz"));
   }
 
   private static GcsOptions gcsOptionsWithTestCredential() {

From 66e564e507b1ad5b9e379e043a193b556e1f3e76 Mon Sep 17 00:00:00 2001
From: Borisa Zivkovic <borisa.zivkovic@huawei.com>
Date: Mon, 15 May 2017 09:24:10 +0100
Subject: [PATCH 253/387] [BEAM-1702] Document how to use BigtableIO with local
 emulator

---
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java  | 26 +++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 22e9f3611ec7..1692cda65623 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -139,6 +139,30 @@
  *         .withTableId("table"));
  * }</pre>
  *
+ * <h3>Using local emulator</h3>
+ *
+ * <p>In order to use local emulator for Bigtable you should use:
+ *
+ * <pre>{@code
+ * BigtableOptions.Builder optionsBuilder =
+ *     new BigtableOptions.Builder()
+ *         .setProjectId("project")
+ *         .setInstanceId("instance")
+ *         .setUsePlaintextNegotiation(true)
+ *         .setCredentialOptions(CredentialOptions.nullCredential())
+ *         .setDataHost("127.0.0.1") // network interface where Bigtable emulator is bound
+ *         .setInstanceAdminHost("127.0.0.1")
+ *         .setTableAdminHost("127.0.0.1")
+ *         .setPort(LOCAL_EMULATOR_PORT))
+ *
+ * PCollection<KV<ByteString, Iterable<Mutation>>> data = ...;
+ *
+ * data.apply("write",
+ *     BigtableIO.write()
+ *         .withBigtableOptions(optionsBuilder)
+ *         .withTableId("table");
+ * }</pre>
+ *
  * <h3>Experimental</h3>
  *
  * <p>This connector for Cloud Bigtable is considered experimental and may break or receive
@@ -205,7 +229,6 @@ public abstract static class Read extends PTransform<PBegin, PCollection<Row>> {
     @Nullable
     abstract BigtableService getBigtableService();
 
-
     /** Returns the Google Cloud Bigtable instance being read from, and other parameters. */
     @Nullable
     public abstract BigtableOptions getBigtableOptions();
@@ -429,7 +452,6 @@ abstract static class Builder {
      * <p>Does not modify this object.
      */
     public Write withBigtableOptions(BigtableOptions options) {
-      checkNotNull(options, "options");
       return withBigtableOptions(options.toBuilder());
     }
 

From a8d2125e5783a556056e88dad8fe3c0a397920d5 Mon Sep 17 00:00:00 2001
From: huafengw <fvunicorn@gmail.com>
Date: Tue, 9 May 2017 11:21:44 +0800
Subject: [PATCH 254/387] [BEAM-659] WindowFn#isCompatible should provide a
 meaningful reason

---
 .../core/construction/PCollectionsTest.java   | 12 ++++++
 .../direct/WindowEvaluatorFactoryTest.java    | 10 +++++
 .../beam/sdk/testing/StaticWindows.java       | 12 ++++++
 .../transforms/windowing/CalendarWindows.java | 36 ++++++++++++++++++
 .../transforms/windowing/FixedWindows.java    | 11 ++++++
 .../transforms/windowing/GlobalWindows.java   | 11 ++++++
 .../IncompatibleWindowException.java          | 38 +++++++++++++++++++
 .../transforms/windowing/InvalidWindows.java  | 11 ++++++
 .../sdk/transforms/windowing/Sessions.java    | 11 ++++++
 .../transforms/windowing/SlidingWindows.java  | 11 ++++++
 .../sdk/transforms/windowing/WindowFn.java    | 21 ++++++++++
 .../beam/sdk/util/IdentityWindowFn.java       | 11 ++++++
 .../beam/sdk/testing/StaticWindowsTest.java   | 12 ++++++
 .../windowing/CalendarWindowsTest.java        | 24 ++++++++++++
 .../windowing/FixedWindowsTest.java           |  7 ++++
 .../transforms/windowing/SessionsTest.java    | 14 +++++++
 .../windowing/SlidingWindowsTest.java         | 11 ++++++
 .../sdk/transforms/windowing/WindowTest.java  |  8 ++++
 .../sdk/util/IdentitySideInputWindowFn.java   |  4 ++
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java   | 13 +++++++
 20 files changed, 288 insertions(+)
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java

diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
index 2c45cbdb94b3..66700d0d6b30 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
@@ -44,6 +44,7 @@
 import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
@@ -157,6 +158,17 @@ public boolean isCompatible(WindowFn<?, ?> other) {
       return other != null && this.getClass().equals(other.getClass());
     }
 
+    @Override
+    public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+      if (!this.isCompatible(other)) {
+        throw new IncompatibleWindowException(
+            other,
+            String.format(
+                "%s is only compatible with %s.",
+                CustomWindows.class.getSimpleName(), CustomWindows.class.getSimpleName()));
+      }
+    }
+
     @Override
     public Coder<BoundedWindow> windowCoder() {
       return new AtomicCoder<BoundedWindow>() {
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
index a91bab54fae8..96fdfaba8b9f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
@@ -35,6 +35,7 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
@@ -303,6 +304,15 @@ public boolean isCompatible(WindowFn<?, ?> other) {
       return false;
     }
 
+    @Override
+    public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "%s is not compatible with any other %s.",
+              EvaluatorTestWindowFn.class.getSimpleName(), WindowFn.class.getSimpleName()));
+    }
+
     @Override
     public Coder<BoundedWindow> windowCoder() {
       @SuppressWarnings({"unchecked", "rawtypes"}) Coder coder =
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java
index fde16695808e..c11057a5c623 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java
@@ -26,6 +26,7 @@
 import java.util.Objects;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
@@ -96,6 +97,17 @@ public boolean isCompatible(WindowFn<?, ?> other) {
     return Objects.equals(this.windows.get(), that.windows.get());
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    if (!this.isCompatible(other)) {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "Only %s objects with the same window supplier are compatible.",
+              StaticWindows.class.getSimpleName()));
+    }
+  }
+
   @Override
   public Coder<BoundedWindow> windowCoder() {
     return coder;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java
index fada50ad5a4d..989c43133220 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/CalendarWindows.java
@@ -144,6 +144,18 @@ public boolean isCompatible(WindowFn<?, ?> other) {
           && timeZone == that.timeZone;
     }
 
+    @Override
+    public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+      if (!this.isCompatible(other)) {
+        throw new IncompatibleWindowException(
+            other,
+            String.format(
+                "Only %s objects with the same number of days, start date "
+                    + "and time zone are compatible.",
+                DaysWindows.class.getSimpleName()));
+      }
+    }
+
     @Override
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
@@ -244,6 +256,18 @@ public boolean isCompatible(WindowFn<?, ?> other) {
           && timeZone == that.timeZone;
     }
 
+    @Override
+    public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+      if (!this.isCompatible(other)) {
+        throw new IncompatibleWindowException(
+            other,
+            String.format(
+                "Only %s objects with the same number of months, "
+                    + "day of month, start date and time zone are compatible.",
+                MonthsWindows.class.getSimpleName()));
+      }
+    }
+
     @Override
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
@@ -353,6 +377,18 @@ public boolean isCompatible(WindowFn<?, ?> other) {
           && timeZone == that.timeZone;
     }
 
+    @Override
+    public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+      if (!this.isCompatible(other)) {
+        throw new IncompatibleWindowException(
+            other,
+            String.format(
+                "Only %s objects with the same number of years, month of year, "
+                    + "day of month, start date and time zone are compatible.",
+                YearsWindows.class.getSimpleName()));
+      }
+    }
+
     @Override
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java
index 8683a600e204..8b169169a780 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/FixedWindows.java
@@ -101,6 +101,17 @@ public boolean isCompatible(WindowFn<?, ?> other) {
     return this.equals(other);
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    if (!this.isCompatible(other)) {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "Only %s objects with the same size and offset are compatible.",
+              FixedWindows.class.getSimpleName()));
+    }
+  }
+
   public Duration getSize() {
     return size;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
index 400be1fe7095..b49328b20198 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
@@ -40,6 +40,17 @@ public boolean isCompatible(WindowFn<?, ?> o) {
     return o instanceof GlobalWindows;
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    if (!this.isCompatible(other)) {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "%s is only compatible with %s.",
+              GlobalWindows.class.getSimpleName(), GlobalWindows.class.getSimpleName()));
+    }
+  }
+
   @Override
   public Coder<GlobalWindow> windowCoder() {
     return GlobalWindow.Coder.INSTANCE;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java
new file mode 100644
index 000000000000..b7b96add2293
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/IncompatibleWindowException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.windowing;
+
+/**
+ * Exception thrown by {@link WindowFn#verifyCompatibility(WindowFn)} if two compared
+ * WindowFns are not compatible, including the explanation of incompatibility.
+ */
+public class IncompatibleWindowException extends Exception {
+  private WindowFn<?, ?> givenWindowFn;
+  private String reason;
+
+  public IncompatibleWindowException(WindowFn<?, ?> windowFn, String reason) {
+    this.givenWindowFn = windowFn;
+    this.reason = reason;
+  }
+
+  @Override
+  public String getMessage() {
+    String windowFn = givenWindowFn == null ? "null" : givenWindowFn.getClass().getSimpleName();
+    return String.format("The given WindowFn is %s. %s", windowFn, reason);
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
index 92041fc54c73..a8084f42d993 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/InvalidWindows.java
@@ -74,6 +74,17 @@ && getOriginalWindowFn().isCompatible(
             ((InvalidWindows<?>) other).getOriginalWindowFn());
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    if (!this.isCompatible(other)) {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "Only %s objects with the same originalWindowFn are compatible.",
+              InvalidWindows.class.getSimpleName()));
+    }
+  }
+
   @Override
   public WindowMappingFn<W> getDefaultWindowMappingFn() {
     throw new UnsupportedOperationException("InvalidWindows is not allowed in side inputs");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
index 5cc7c65b7da5..115a964740b4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Sessions.java
@@ -79,6 +79,17 @@ public boolean isCompatible(WindowFn<?, ?> other) {
     return other instanceof Sessions;
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    if (!this.isCompatible(other)) {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "%s is only compatible with %s.",
+              Sessions.class.getSimpleName(), Sessions.class.getSimpleName()));
+    }
+  }
+
   @Override
   public WindowMappingFn<IntervalWindow> getDefaultWindowMappingFn() {
     throw new UnsupportedOperationException("Sessions is not allowed in side inputs");
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java
index 650dc37232b0..f65788438021 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/SlidingWindows.java
@@ -147,6 +147,17 @@ public boolean isCompatible(WindowFn<?, ?> other) {
     return equals(other);
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    if (!this.isCompatible(other)) {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "Only %s objects with the same size, period and offset are compatible.",
+              SlidingWindows.class.getSimpleName()));
+    }
+  }
+
   @Override
   public void populateDisplayData(DisplayData.Builder builder) {
     super.populateDisplayData(builder);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
index 5ebbb41b5956..e329c1d1ace8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
@@ -114,9 +114,30 @@ public abstract void merge(Collection<W> toBeMerged, W mergeResult)
   /**
    * Returns whether this performs the same merging as the given
    * {@code WindowFn}.
+   *
+   * @deprecated please override verifyCompatibility to throw a useful error message;
+   *     we will remove isCompatible at version 3.0.0
    */
+  @Deprecated
   public abstract boolean isCompatible(WindowFn<?, ?> other);
 
+  /**
+   * Throw {@link IncompatibleWindowException} if this WindowFn does not perform the same merging as
+   * the given ${@code WindowFn}.
+   *
+   * @throws IncompatibleWindowException if compared WindowFns are not compatible.
+   */
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    if (!this.isCompatible(other)) {
+      throw new IncompatibleWindowException(
+          other,
+          String.format(
+              "%s is not compatible with %s",
+              this.getClass().getSimpleName(),
+              other.getClass().getSimpleName()));
+    }
+  }
+
   /**
    * Returns the {@link Coder} used for serializing the windows used
    * by this windowFn.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
index a61e3a6482dc..a4bfdda4f813 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/IdentityWindowFn.java
@@ -23,6 +23,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -83,6 +84,16 @@ public boolean isCompatible(WindowFn<?, ?> other) {
             getClass().getCanonicalName()));
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+    throw new UnsupportedOperationException(
+        String.format(
+            "%s.verifyCompatibility() should never be called."
+                + " It is a private implementation detail of sdk utilities."
+                + " This message indicates a bug in the Beam SDK.",
+            getClass().getCanonicalName()));
+  }
+
   @Override
   public Coder<BoundedWindow> windowCoder() {
     // Safe because the prior WindowFn provides both the windows and the coder.
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java
index 7ee48c8c1ecb..2969ca66b362 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java
@@ -22,6 +22,7 @@
 import com.google.common.collect.ImmutableList;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.hamcrest.Matchers;
@@ -93,4 +94,15 @@ public void emptyIterableThrows() {
     thrown.expectMessage("may not be empty");
     StaticWindows.of(GlobalWindow.Coder.INSTANCE, ImmutableList.<GlobalWindow>of());
   }
+
+  @Test
+  public void testCompatibility() throws IncompatibleWindowException {
+    StaticWindows staticWindows =
+        StaticWindows.of(IntervalWindow.getCoder(), ImmutableList.of(first, second));
+    staticWindows.verifyCompatibility(
+        StaticWindows.of(IntervalWindow.getCoder(), ImmutableList.of(first, second)));
+    thrown.expect(IncompatibleWindowException.class);
+    staticWindows.verifyCompatibility(
+        StaticWindows.of(IntervalWindow.getCoder(), ImmutableList.of(first)));
+  }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java
index cd562e9bea17..c8c01f5e4882 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/CalendarWindowsTest.java
@@ -90,6 +90,14 @@ public void testDays() throws Exception {
     assertEquals(expected, runWindowFn(CalendarWindows.days(1), timestamps));
   }
 
+  @Test
+  public void testDaysCompatibility() throws IncompatibleWindowException {
+    CalendarWindows.DaysWindows daysWindows = CalendarWindows.days(10);
+    daysWindows.verifyCompatibility(CalendarWindows.days(10));
+    thrown.expect(IncompatibleWindowException.class);
+    daysWindows.verifyCompatibility(CalendarWindows.days(9));
+  }
+
   @Test
   public void testWeeks() throws Exception {
     Map<IntervalWindow, Set<String>> expected = new HashMap<>();
@@ -164,6 +172,14 @@ public void testMonths() throws Exception {
         runWindowFn(CalendarWindows.months(1), timestamps));
   }
 
+  @Test
+  public void testMonthsCompatibility() throws IncompatibleWindowException {
+    CalendarWindows.MonthsWindows monthsWindows = CalendarWindows.months(10).beginningOnDay(15);
+    monthsWindows.verifyCompatibility(CalendarWindows.months(10).beginningOnDay(15));
+    thrown.expect(IncompatibleWindowException.class);
+    monthsWindows.verifyCompatibility(CalendarWindows.months(10).beginningOnDay(30));
+  }
+
   @Test
   public void testMultiMonths() throws Exception {
     Map<IntervalWindow, Set<String>> expected = new HashMap<>();
@@ -238,6 +254,14 @@ public void testYears() throws Exception {
         CalendarWindows.years(10).withStartingYear(2000).beginningOnDay(5, 5), timestamps));
   }
 
+  @Test
+  public void testYearsCompatibility() throws IncompatibleWindowException {
+    CalendarWindows.YearsWindows yearsWindows = CalendarWindows.years(2017).beginningOnDay(1, 1);
+    yearsWindows.verifyCompatibility(CalendarWindows.years(2017).beginningOnDay(1, 1));
+    thrown.expect(IncompatibleWindowException.class);
+    yearsWindows.verifyCompatibility(CalendarWindows.years(2017).beginningOnDay(1, 2));
+  }
+
   @Test
   public void testTimeZone() throws Exception {
     Map<IntervalWindow, Set<String>> expected = new HashMap<>();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java
index 47c273a13c54..80a534c5550a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/FixedWindowsTest.java
@@ -148,6 +148,13 @@ public void testEquality() {
         FixedWindows.of(new Duration(20))));
   }
 
+  @Test
+  public void testVerifyCompatibility() throws IncompatibleWindowException {
+    FixedWindows.of(new Duration(10)).verifyCompatibility(FixedWindows.of(new Duration(10)));
+    thrown.expect(IncompatibleWindowException.class);
+    FixedWindows.of(new Duration(10)).verifyCompatibility(FixedWindows.of(new Duration(20)));
+  }
+
   @Test
   public void testValidOutputTimes() throws Exception {
     for (long timestamp : Arrays.asList(200, 800, 700)) {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java
index 9d949286e02e..42c15b5233a3 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java
@@ -36,7 +36,9 @@
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -45,6 +47,8 @@
  */
 @RunWith(JUnit4.class)
 public class SessionsTest {
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
 
   @Test
   public void testSimple() throws Exception {
@@ -106,6 +110,16 @@ public void testEquality() {
             Sessions.withGapDuration(new Duration(20))));
   }
 
+  @Test
+  public void testVerifyCompatibility() throws IncompatibleWindowException {
+    Sessions.withGapDuration(new Duration(10))
+        .verifyCompatibility(Sessions.withGapDuration(new Duration(10)));
+
+    thrown.expect(IncompatibleWindowException.class);
+    Sessions.withGapDuration(new Duration(10))
+        .verifyCompatibility(FixedWindows.of(new Duration(10)));
+  }
+
   /**
    * Validates that the output timestamp for aggregate data falls within the acceptable range.
    */
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java
index dd673d318b20..b14e2215173d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SlidingWindowsTest.java
@@ -34,7 +34,9 @@
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -43,6 +45,8 @@
  */
 @RunWith(JUnit4.class)
 public class SlidingWindowsTest {
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
 
   @Test
   public void testSimple() throws Exception {
@@ -152,6 +156,13 @@ public void testEquality() {
         SlidingWindows.of(new Duration(20))));
   }
 
+  @Test
+  public void testVerifyCompatibility() throws IncompatibleWindowException {
+    SlidingWindows.of(new Duration(10)).verifyCompatibility(SlidingWindows.of(new Duration(10)));
+    thrown.expect(IncompatibleWindowException.class);
+    SlidingWindows.of(new Duration(10)).verifyCompatibility(SlidingWindows.of(new Duration(20)));
+  }
+
   @Test
   public void testDefaultWindowMappingFn() {
     // [40, 1040), [340, 1340), [640, 1640) ...
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 92f6a9c01457..f536a9a9b8a1 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -300,6 +300,14 @@ public boolean isCompatible(WindowFn<?, ?> other) {
       return other instanceof WindowOddEvenBuckets;
     }
 
+    @Override
+    public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+      if (!this.isCompatible(other)) {
+        throw new IncompatibleWindowException(
+            other, "WindowOddEvenBuckets is only compatible with WindowOddEvenBuckets.");
+      }
+    }
+
     @Override
     public Coder<IntervalWindow> windowCoder() {
       return new IntervalWindow.IntervalWindowCoder();
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java
index 2171466b1192..32e23dab142f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/IdentitySideInputWindowFn.java
@@ -22,6 +22,7 @@
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
@@ -42,6 +43,9 @@ public boolean isCompatible(WindowFn<?, ?> other) {
     return true;
   }
 
+  @Override
+  public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {}
+
   @Override
   public Coder<BoundedWindow> windowCoder() {
     // not used
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index d60c7214b701..ba0cea81127c 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -115,6 +115,7 @@
 import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -697,6 +698,18 @@ public boolean isCompatible(WindowFn<?, ?> o) {
       return o instanceof PartitionedGlobalWindows;
     }
 
+    @Override
+    public void verifyCompatibility(WindowFn<?, ?> other) throws IncompatibleWindowException {
+      if (!this.isCompatible(other)) {
+        throw new IncompatibleWindowException(
+            other,
+            String.format(
+                "%s is only compatible with %s.",
+                PartitionedGlobalWindows.class.getSimpleName(),
+                PartitionedGlobalWindows.class.getSimpleName()));
+      }
+    }
+
     @Override
     public Coder<PartitionedGlobalWindow> windowCoder() {
       return new PartitionedGlobalWindowCoder();

From 26a615e885b39f92dc81d383c8115906c280187a Mon Sep 17 00:00:00 2001
From: Andrew Martin <andrewsmartin.mg@gmail.com>
Date: Thu, 18 May 2017 15:52:09 -0400
Subject: [PATCH 255/387] Remove incorrect `Message` bound on writeAvros

---
 .../java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java     | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index e293b95ede1e..4f33d61ae403 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -527,7 +527,7 @@ public static <T extends Message> Write<T> writeProtos(Class<T> messageClass) {
    * Returns A {@link PTransform} that writes binary encoded Avro messages of a given type
    * to a Google Cloud Pub/Sub stream.
    */
-  public static <T extends Message> Write<T> writeAvros(Class<T> clazz) {
+  public static <T> Write<T> writeAvros(Class<T> clazz) {
     // TODO: Like in readAvros(), stop using AvroCoder and instead format the payload directly.
     return PubsubIO.<T>write().withFormatFn(new FormatPayloadUsingCoder<>(AvroCoder.of(clazz)));
   }
@@ -970,8 +970,7 @@ public PubsubMessage apply(String input) {
     }
   }
 
-  private static class FormatPayloadUsingCoder<T extends Message>
-      extends SimpleFunction<T, PubsubMessage> {
+  private static class FormatPayloadUsingCoder<T> extends SimpleFunction<T, PubsubMessage> {
     private Coder<T> coder;
 
     public FormatPayloadUsingCoder(Coder<T> coder) {

From 790e7fe6653b926044d3dfecdccbc2fda9c998f0 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 21 Mar 2017 15:06:58 -0700
Subject: [PATCH 256/387] Add ParDos

Add ParDoPayloadTranslator to PTransformTranslator
---
 .../core/construction/PTransforms.java        |   8 +-
 .../runners/core/construction/ParDos.java     | 317 ++++++++++++++++++
 .../construction/RunnerPCollectionView.java   |  88 +++++
 .../runners/core/construction/ParDosTest.java | 229 +++++++++++++
 .../src/main/proto/beam_runner_api.proto      |  10 +
 .../transforms/windowing/GlobalWindows.java   |   6 +-
 6 files changed, 653 insertions(+), 5 deletions(-)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
 create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
index d25d342bfd65..16276b9ed82c 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
@@ -24,10 +24,12 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import org.apache.beam.runners.core.construction.ParDos.ParDoPayloadTranslator;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
@@ -39,9 +41,9 @@
 public class PTransforms {
   private static final Map<Class<? extends PTransform>, TransformPayloadTranslator>
       KNOWN_PAYLOAD_TRANSLATORS =
-          ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder().build();
-  // TODO: ParDoPayload, WindowIntoPayload, ReadPayload, CombinePayload
-  // TODO: "Flatten Payload", etc?
+          ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder()
+              .put(ParDo.MultiOutput.class, ParDoPayloadTranslator.create())
+              .build();
   // TODO: Load via service loader.
   private PTransforms() {}
 
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
new file mode 100644
index 000000000000..b2b29df65dc0
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Optional;
+import com.google.protobuf.Any;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.BytesValue;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.runners.core.construction.PTransforms.TransformPayloadTranslator;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.Parameter.Type;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput.Builder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.StateSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.TimerSpec;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Materializations;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
+import org.apache.beam.sdk.transforms.ViewFn;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.Cases;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.StateDeclaration;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerDeclaration;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/**
+ * Utilities for interacting with {@link ParDo} instances and {@link ParDoPayload} protos.
+ */
+public class ParDos {
+  /**
+   * The URN for a {@link ParDoPayload}.
+   */
+  public static final String PAR_DO_PAYLOAD_URN = "urn:beam:pardo:v1";
+  /**
+   * The URN for an unknown Java {@link DoFn}.
+   */
+  public static final String CUSTOM_JAVA_DO_FN_URN = "urn:beam:dofn:javasdk:0.1";
+  /**
+   * The URN for an unknown Java {@link ViewFn}.
+   */
+  public static final String CUSTOM_JAVA_VIEW_FN_URN = "urn:beam:viewfn:javasdk:0.1";
+  /**
+   * The URN for an unknown Java {@link WindowMappingFn}.
+   */
+  public static final String CUSTOM_JAVA_WINDOW_MAPPING_FN_URN =
+      "urn:beam:windowmappingfn:javasdk:0.1";
+
+  /**
+   * A {@link TransformPayloadTranslator} for {@link ParDo}.
+   */
+  public static class ParDoPayloadTranslator
+      implements PTransforms.TransformPayloadTranslator<ParDo.MultiOutput<?, ?>> {
+    public static TransformPayloadTranslator create() {
+      return new ParDoPayloadTranslator();
+    }
+
+    private ParDoPayloadTranslator() {}
+
+    @Override
+    public FunctionSpec translate(
+        AppliedPTransform<?, ?, MultiOutput<?, ?>> transform, SdkComponents components) {
+      ParDoPayload payload = toProto(transform.getTransform(), components);
+      return RunnerApi.FunctionSpec.newBuilder()
+          .setUrn(PAR_DO_PAYLOAD_URN)
+          .setParameter(Any.pack(payload))
+          .build();
+    }
+  }
+
+  public static ParDoPayload toProto(ParDo.MultiOutput<?, ?> parDo, SdkComponents components) {
+    DoFnSignature signature = DoFnSignatures.getSignature(parDo.getFn().getClass());
+    Map<String, StateDeclaration> states = signature.stateDeclarations();
+    Map<String, TimerDeclaration> timers = signature.timerDeclarations();
+    List<Parameter> parameters = signature.processElement().extraParameters();
+
+    ParDoPayload.Builder builder = ParDoPayload.newBuilder();
+    builder.setDoFn(toProto(parDo.getFn(), parDo.getMainOutputTag()));
+    for (PCollectionView<?> sideInput : parDo.getSideInputs()) {
+      builder.putSideInputs(sideInput.getTagInternal().getId(), toProto(sideInput));
+    }
+    for (Parameter parameter : parameters) {
+      Optional<RunnerApi.Parameter> protoParameter = toProto(parameter);
+      if (protoParameter.isPresent()) {
+        builder.addParameters(protoParameter.get());
+      }
+    }
+    for (Map.Entry<String, StateDeclaration> state : states.entrySet()) {
+      StateSpec spec = toProto(state.getValue());
+      builder.putStateSpecs(state.getKey(), spec);
+    }
+    for (Map.Entry<String, TimerDeclaration> timer : timers.entrySet()) {
+      TimerSpec spec = toProto(timer.getValue());
+      builder.putTimerSpecs(timer.getKey(), spec);
+    }
+    return builder.build();
+  }
+
+  public static DoFn<?, ?> getDoFn(ParDoPayload payload) throws InvalidProtocolBufferException {
+    return doFnAndMainOutputTagFromProto(payload.getDoFn()).getDoFn();
+  }
+
+  public static TupleTag<?> getMainOutputTag(ParDoPayload payload)
+      throws InvalidProtocolBufferException {
+    return doFnAndMainOutputTagFromProto(payload.getDoFn()).getMainOutputTag();
+  }
+
+  // TODO: Implement
+  private static StateSpec toProto(StateDeclaration state) {
+    throw new UnsupportedOperationException("Not yet supported");
+  }
+
+  // TODO: Implement
+  private static TimerSpec toProto(TimerDeclaration timer) {
+    throw new UnsupportedOperationException("Not yet supported");
+  }
+
+  @AutoValue
+  abstract static class DoFnAndMainOutput implements Serializable {
+    public static DoFnAndMainOutput of(
+        DoFn<?, ?> fn, TupleTag<?> tag) {
+      return new AutoValue_ParDos_DoFnAndMainOutput(fn, tag);
+    }
+
+    abstract DoFn<?, ?> getDoFn();
+    abstract TupleTag<?> getMainOutputTag();
+  }
+
+  private static SdkFunctionSpec toProto(DoFn<?, ?> fn, TupleTag<?> tag) {
+    return SdkFunctionSpec.newBuilder()
+        .setSpec(
+            FunctionSpec.newBuilder()
+                .setUrn(CUSTOM_JAVA_DO_FN_URN)
+                .setParameter(
+                    Any.pack(
+                        BytesValue.newBuilder()
+                            .setValue(
+                                ByteString.copyFrom(
+                                    SerializableUtils.serializeToByteArray(
+                                        DoFnAndMainOutput.of(fn, tag))))
+                            .build())))
+        .build();
+  }
+
+  private static DoFnAndMainOutput doFnAndMainOutputTagFromProto(SdkFunctionSpec fnSpec)
+      throws InvalidProtocolBufferException {
+    checkArgument(fnSpec.getSpec().getUrn().equals(CUSTOM_JAVA_DO_FN_URN));
+    byte[] serializedFn =
+        fnSpec.getSpec().getParameter().unpack(BytesValue.class).getValue().toByteArray();
+    return (DoFnAndMainOutput)
+        SerializableUtils.deserializeFromByteArray(serializedFn, "Custom DoFn And Main Output tag");
+  }
+
+  private static Optional<RunnerApi.Parameter> toProto(Parameter parameter) {
+    return parameter.match(
+        new Cases.WithDefault<Optional<RunnerApi.Parameter>>() {
+          @Override
+          public Optional<RunnerApi.Parameter> dispatch(WindowParameter p) {
+            return Optional.of(RunnerApi.Parameter.newBuilder().setType(Type.WINDOW).build());
+          }
+
+          @Override
+          public Optional<RunnerApi.Parameter> dispatch(RestrictionTrackerParameter p) {
+            return Optional.of(
+                RunnerApi.Parameter.newBuilder().setType(Type.RESTRICTION_TRACKER).build());
+          }
+
+          @Override
+          protected Optional<RunnerApi.Parameter> dispatchDefault(Parameter p) {
+            return Optional.absent();
+          }
+        });
+  }
+
+  private static SideInput toProto(PCollectionView<?> view) {
+    Builder builder = SideInput.newBuilder();
+    builder.setAccessPattern(
+        FunctionSpec.newBuilder()
+            .setUrn(view.getViewFn().getMaterialization().getUrn())
+            .build());
+    builder.setViewFn(toProto(view.getViewFn()));
+    builder.setWindowMappingFn(toProto(view.getWindowMappingFn()));
+    return builder.build();
+  }
+
+  public static PCollectionView<?> fromProto(
+      SideInput sideInput, String id, RunnerApi.PTransform parDoTransform, Components components)
+      throws IOException {
+    TupleTag<?> tag = new TupleTag<>(id);
+    WindowMappingFn<?> windowMappingFn = windowMappingFnFromProto(sideInput.getWindowMappingFn());
+    ViewFn<?, ?> viewFn = viewFnFromProto(sideInput.getViewFn());
+
+    RunnerApi.PCollection inputCollection =
+        components.getPcollectionsOrThrow(parDoTransform.getInputsOrThrow(id));
+    WindowingStrategy<?, ?> windowingStrategy =
+        WindowingStrategies.fromProto(
+            components.getWindowingStrategiesOrThrow(inputCollection.getWindowingStrategyId()),
+            components);
+    Coder<?> elemCoder =
+        Coders.fromProto(components.getCodersOrThrow(inputCollection.getCoderId()), components);
+    Coder<Iterable<WindowedValue<?>>> coder =
+        (Coder)
+            IterableCoder.of(
+                FullWindowedValueCoder.of(
+                    elemCoder, windowingStrategy.getWindowFn().windowCoder()));
+    checkArgument(
+        sideInput.getAccessPattern().getUrn().equals(Materializations.ITERABLE_MATERIALIZATION_URN),
+        "Unknown View Materialization URN %s",
+        sideInput.getAccessPattern().getUrn());
+
+    PCollectionView<?> view =
+        new RunnerPCollectionView<>(
+            (TupleTag<Iterable<WindowedValue<?>>>) tag,
+            (ViewFn<Iterable<WindowedValue<?>>, ?>) viewFn,
+            windowMappingFn,
+            windowingStrategy,
+            coder);
+    return view;
+  }
+
+  private static SdkFunctionSpec toProto(ViewFn<?, ?> viewFn) {
+    return SdkFunctionSpec.newBuilder()
+        .setSpec(
+            FunctionSpec.newBuilder()
+                .setUrn(CUSTOM_JAVA_VIEW_FN_URN)
+                .setParameter(
+                    Any.pack(
+                        BytesValue.newBuilder()
+                            .setValue(
+                                ByteString.copyFrom(SerializableUtils.serializeToByteArray(viewFn)))
+                            .build())))
+        .build();
+  }
+
+  private static ViewFn<?, ?> viewFnFromProto(SdkFunctionSpec viewFn)
+      throws InvalidProtocolBufferException {
+    FunctionSpec spec = viewFn.getSpec();
+    checkArgument(
+        spec.getUrn().equals(CUSTOM_JAVA_VIEW_FN_URN),
+        "Can't deserialize unknown %s type %s",
+        ViewFn.class.getSimpleName(),
+        spec.getUrn());
+    return (ViewFn<?, ?>)
+        SerializableUtils.deserializeFromByteArray(
+            spec.getParameter().unpack(BytesValue.class).getValue().toByteArray(), "Custom ViewFn");
+  }
+
+  private static SdkFunctionSpec toProto(WindowMappingFn<?> windowMappingFn) {
+    return SdkFunctionSpec.newBuilder()
+        .setSpec(
+            FunctionSpec.newBuilder()
+                .setUrn(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN)
+                .setParameter(
+                    Any.pack(
+                        BytesValue.newBuilder()
+                            .setValue(
+                                ByteString.copyFrom(
+                                    SerializableUtils.serializeToByteArray(windowMappingFn)))
+                            .build())))
+        .build();
+  }
+
+  private static WindowMappingFn<?> windowMappingFnFromProto(SdkFunctionSpec windowMappingFn)
+      throws InvalidProtocolBufferException {
+    FunctionSpec spec = windowMappingFn.getSpec();
+    checkArgument(
+        spec.getUrn().equals(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN),
+        "Can't deserialize unknown %s type %s",
+        WindowMappingFn.class.getSimpleName(),
+        spec.getUrn());
+    return (WindowMappingFn<?>)
+        SerializableUtils.deserializeFromByteArray(
+            spec.getParameter().unpack(BytesValue.class).getValue().toByteArray(),
+            "Custom WinodwMappingFn");
+  }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
new file mode 100644
index 000000000000..89e878496ef3
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
+import org.apache.beam.sdk.transforms.ViewFn;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValueBase;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/** A {@link PCollectionView} created from the components of a {@link SideInput}. */
+class RunnerPCollectionView<T> extends PValueBase implements PCollectionView<T> {
+  private final TupleTag<Iterable<WindowedValue<?>>> tag;
+  private final ViewFn<Iterable<WindowedValue<?>>, T> viewFn;
+  private final WindowMappingFn<?> windowMappingFn;
+  private final WindowingStrategy<?, ?> windowingStrategy;
+  private final Coder<Iterable<WindowedValue<?>>> coder;
+
+  /**
+   * Create a new {@link RunnerPCollectionView} from the provided components.
+   */
+  RunnerPCollectionView(
+      TupleTag<Iterable<WindowedValue<?>>> tag,
+      ViewFn<Iterable<WindowedValue<?>>, T> viewFn,
+      WindowMappingFn<?> windowMappingFn,
+      @Nullable WindowingStrategy<?, ?> windowingStrategy,
+      @Nullable Coder<Iterable<WindowedValue<?>>> coder) {
+    this.tag = tag;
+    this.viewFn = viewFn;
+    this.windowMappingFn = windowMappingFn;
+    this.windowingStrategy = windowingStrategy;
+    this.coder = coder;
+  }
+
+  @Nullable
+  @Override
+  public PCollection<?> getPCollection() {
+    throw new IllegalStateException(
+        String.format("Cannot call getPCollection on a %s", getClass().getSimpleName()));
+  }
+
+  @Override
+  public TupleTag<Iterable<WindowedValue<?>>> getTagInternal() {
+    return tag;
+  }
+
+  @Override
+  public ViewFn<Iterable<WindowedValue<?>>, T> getViewFn() {
+    return viewFn;
+  }
+
+  @Override
+  public WindowMappingFn<?> getWindowMappingFn() {
+    return windowMappingFn;
+  }
+
+  @Override
+  public WindowingStrategy<?, ?> getWindowingStrategyInternal() {
+    return windowingStrategy;
+  }
+
+  @Override
+  public Coder<Iterable<WindowedValue<?>>> getCoderInternal() {
+    return coder;
+  }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java
new file mode 100644
index 000000000000..74edec15af81
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.state.BagState;
+import org.apache.beam.sdk.state.CombiningState;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.Timer;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.state.TimerSpecs;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine.BinaryCombineLongFn;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Tests for {@link ParDos}. */
+@RunWith(Parameterized.class)
+public class ParDosTest {
+  public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+  private static PCollectionView<Long> singletonSideInput =
+      p.apply("GenerateSingleton", GenerateSequence.from(0L).to(1L))
+          .apply(View.<Long>asSingleton());
+  private static PCollectionView<Map<Long, Iterable<String>>> multimapSideInput =
+      p.apply("CreateMultimap", Create.of(KV.of(1L, "foo"), KV.of(1L, "bar"), KV.of(2L, "spam")))
+          .setCoder(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of()))
+          .apply(View.<Long, String>asMultimap());
+
+  private static PCollection<KV<Long, String>> mainInput =
+      p.apply("CreateMainInput", Create.empty(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of())));
+
+  @Parameters(name = "{index}: {0}")
+  public static Iterable<ParDo.MultiOutput<?, ?>> data() {
+    return ImmutableList.<ParDo.MultiOutput<?, ?>>of(
+        ParDo.of(new DropElementsFn()).withOutputTags(new TupleTag<Void>(), TupleTagList.empty()),
+        ParDo.of(new DropElementsFn())
+            .withOutputTags(new TupleTag<Void>(), TupleTagList.empty())
+            .withSideInputs(singletonSideInput, multimapSideInput),
+        ParDo.of(new DropElementsFn())
+            .withOutputTags(
+                new TupleTag<Void>(),
+                TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {}))
+            .withSideInputs(singletonSideInput, multimapSideInput),
+        ParDo.of(new DropElementsFn())
+            .withOutputTags(
+                new TupleTag<Void>(),
+                TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {})));
+  }
+
+  @Parameter(0)
+  public ParDo.MultiOutput<KV<Long, String>, Void> parDo;
+
+  @Test
+  public void testToAndFromProto() throws Exception {
+    SdkComponents components = SdkComponents.create();
+    ParDoPayload payload = ParDos.toProto(parDo, components);
+
+    assertThat(ParDos.getDoFn(payload), Matchers.<DoFn<?, ?>>equalTo(parDo.getFn()));
+    assertThat(
+        ParDos.getMainOutputTag(payload), Matchers.<TupleTag<?>>equalTo(parDo.getMainOutputTag()));
+    for (PCollectionView<?> view : parDo.getSideInputs()) {
+      payload.getSideInputsOrThrow(view.getTagInternal().getId());
+    }
+  }
+
+  @Test
+  public void toAndFromTransformProto() throws Exception {
+    Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+    inputs.put(new TupleTag<KV<Long, String>>() {}, mainInput);
+    inputs.putAll(parDo.getAdditionalInputs());
+    PCollectionTuple output = mainInput.apply(parDo);
+
+    SdkComponents components = SdkComponents.create();
+    String transformId =
+        components.registerPTransform(
+            AppliedPTransform.<PCollection<KV<Long, String>>, PCollection<Void>, MultiOutput>of(
+                "foo", inputs, output.expand(), parDo, p),
+            Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+
+    Components protoComponents = components.toComponents();
+    RunnerApi.PTransform protoTransform =
+        protoComponents.getTransformsOrThrow(transformId);
+    ParDoPayload parDoPayload = protoTransform.getSpec().getParameter().unpack(ParDoPayload.class);
+    for (PCollectionView<?> view : parDo.getSideInputs()) {
+      SideInput sideInput = parDoPayload.getSideInputsOrThrow(view.getTagInternal().getId());
+      PCollectionView<?> restoredView =
+          ParDos.fromProto(
+              sideInput, view.getTagInternal().getId(), protoTransform, protoComponents);
+      assertThat(restoredView.getTagInternal(), equalTo(view.getTagInternal()));
+      assertThat(restoredView.getViewFn(), instanceOf(view.getViewFn().getClass()));
+      assertThat(
+          restoredView.getWindowMappingFn(), instanceOf(view.getWindowMappingFn().getClass()));
+      assertThat(
+          restoredView.getWindowingStrategyInternal(),
+          Matchers.<WindowingStrategy<?, ?>>equalTo(
+              view.getWindowingStrategyInternal().fixDefaults()));
+      assertThat(restoredView.getCoderInternal(), equalTo(view.getCoderInternal()));
+    }
+  }
+
+  private static class DropElementsFn extends DoFn<KV<Long, String>, Void> {
+    @ProcessElement
+    public void proc(ProcessContext context, BoundedWindow window) {
+      context.output(null);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      return other instanceof DropElementsFn;
+    }
+
+    @Override
+    public int hashCode() {
+      return DropElementsFn.class.hashCode();
+    }
+  }
+
+  @SuppressWarnings("unused")
+  private static class StateTimerDropElementsFn extends DoFn<KV<Long, String>, Void> {
+    private static final String BAG_STATE_ID = "bagState";
+    private static final String COMBINING_STATE_ID = "combiningState";
+    private static final String EVENT_TIMER_ID = "eventTimer";
+    private static final String PROCESSING_TIMER_ID = "processingTimer";
+
+    @StateId(BAG_STATE_ID)
+    private final StateSpec<BagState<String>> bagState = StateSpecs.bag(StringUtf8Coder.of());
+
+    @StateId(COMBINING_STATE_ID)
+    private final StateSpec<CombiningState<Long, long[], Long>> combiningState =
+        StateSpecs.combining(
+            new BinaryCombineLongFn() {
+              @Override
+              public long apply(long left, long right) {
+                return Math.max(left, right);
+              }
+
+              @Override
+              public long identity() {
+                return Long.MIN_VALUE;
+              }
+            });
+
+    @TimerId(EVENT_TIMER_ID)
+    private final TimerSpec eventTimer = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+    @TimerId(PROCESSING_TIMER_ID)
+    private final TimerSpec processingTimer = TimerSpecs.timer(TimeDomain.PROCESSING_TIME);
+
+    @ProcessElement
+    public void dropInput(
+        ProcessContext context,
+        BoundedWindow window,
+        @StateId(BAG_STATE_ID) BagState<String> bagStateState,
+        @StateId(COMBINING_STATE_ID) CombiningState<Long, long[], Long> combiningStateState,
+        @TimerId(EVENT_TIMER_ID) Timer eventTimerTimer,
+        @TimerId(PROCESSING_TIMER_ID) Timer processingTimerTimer) {
+      context.output(null);
+    }
+
+    @OnTimer(EVENT_TIMER_ID)
+    public void onEventTime(OnTimerContext context) {}
+
+    @OnTimer(PROCESSING_TIMER_ID)
+    public void onProcessingTime(OnTimerContext context) {}
+
+    @Override
+    public boolean equals(Object other) {
+      return other instanceof StateTimerDropElementsFn;
+    }
+
+    @Override
+    public int hashCode() {
+      return StateTimerDropElementsFn.class.hashCode();
+    }
+  }
+}
diff --git a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
index bf4df2af65fd..c8722e6a39fb 100644
--- a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
+++ b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
@@ -248,10 +248,20 @@ message Parameter {
 
 message StateSpec {
   // TODO: AST for state spec
+  string id = 1;
+  Type type = 2;
+
+  enum Type {
+    VALUE = 0;
+    BAG = 1;
+    MAP = 2;
+    SET = 3;
+  }
 }
 
 message TimerSpec {
   // TODO: AST for timer spec
+  string id = 1;
 }
 
 enum IsBounded {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
index 1103a24a8af1..d48d26b1807c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindows.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.transforms.windowing;
 
+import com.google.auto.value.AutoValue;
 import java.util.Collection;
 import java.util.Collections;
 import org.apache.beam.sdk.coders.Coder;
@@ -61,10 +62,11 @@ public Coder<GlobalWindow> windowCoder() {
 
   @Override
   public WindowMappingFn<GlobalWindow> getDefaultWindowMappingFn() {
-    return new GlobalWindowMappingFn();
+    return new AutoValue_GlobalWindows_GlobalWindowMappingFn();
   }
 
-  static class GlobalWindowMappingFn extends WindowMappingFn<GlobalWindow> {
+  @AutoValue
+  abstract static class GlobalWindowMappingFn extends WindowMappingFn<GlobalWindow> {
     @Override
     public GlobalWindow getSideInputWindow(BoundedWindow mainWindow) {
       return GlobalWindow.INSTANCE;

From 7a2e829c849d3828353994b8eddca4784a900aca Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Mon, 15 May 2017 15:54:03 -0700
Subject: [PATCH 257/387] Splits SplittableParDo into a core-construction part
 and a KWI-related part

---
 .../apache/beam/runners/apex/ApexRunner.java  |   7 +-
 .../translation/ApexPipelineTranslator.java   |  11 +-
 .../apex/translation/ParDoTranslator.java     |  11 +-
 .../operators/ApexParDoOperator.java          |  10 +-
 .../construction}/ElementAndRestriction.java  |   2 +-
 .../ElementAndRestrictionCoder.java           |   2 +-
 .../core/construction/SplittableParDo.java    | 298 ++++++++++++++
 .../ElementAndRestrictionCoderTest.java       |   5 +-
 .../construction/SplittableParDoTest.java     | 157 ++++++++
 .../apache/beam/runners/core/DoFnRunners.java |   2 +-
 .../beam/runners/core/ProcessFnRunner.java    |   7 +-
 ... => SplittableParDoViaKeyedWorkItems.java} | 372 +++++-------------
 ...java => SplittableParDoProcessFnTest.java} | 107 +----
 ...tGBKIntoKeyedWorkItemsOverrideFactory.java |   9 +-
 .../beam/runners/direct/DirectRunner.java     |  10 +-
 .../direct/KeyedPValueTrackingVisitor.java    |   4 +-
 .../direct/ParDoMultiOverrideFactory.java     |   2 +-
 ...ttableProcessElementsEvaluatorFactory.java |  11 +-
 .../direct/TransformEvaluatorRegistry.java    |   4 +-
 .../FlinkStreamingPipelineTranslator.java     |   7 +-
 .../FlinkStreamingTransformTranslators.java   |  29 +-
 .../streaming/SplittableDoFnOperator.java     |  12 +-
 22 files changed, 638 insertions(+), 441 deletions(-)
 rename runners/{core-java/src/main/java/org/apache/beam/runners/core => core-construction-java/src/main/java/org/apache/beam/runners/core/construction}/ElementAndRestriction.java (97%)
 rename runners/{core-java/src/main/java/org/apache/beam/runners/core => core-construction-java/src/main/java/org/apache/beam/runners/core/construction}/ElementAndRestrictionCoder.java (98%)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
 rename runners/{core-java/src/test/java/org/apache/beam/runners/core => core-construction-java/src/test/java/org/apache/beam/runners/core/construction}/ElementAndRestrictionCoderTest.java (96%)
 create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
 rename runners/core-java/src/main/java/org/apache/beam/runners/core/{SplittableParDo.java => SplittableParDoViaKeyedWorkItems.java} (53%)
 rename runners/core-java/src/test/java/org/apache/beam/runners/core/{SplittableParDoTest.java => SplittableParDoProcessFnTest.java} (82%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index 2fd0b22363f5..f91d8e550789 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -39,12 +39,13 @@
 import org.apache.apex.api.Launcher.AppHandle;
 import org.apache.apex.api.Launcher.LaunchMode;
 import org.apache.beam.runners.apex.translation.ApexPipelineTranslator;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
 import org.apache.beam.runners.core.construction.PTransformMatchers;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.PrimitiveCreate;
 import org.apache.beam.runners.core.construction.ReplacementOutputs;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
+import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
@@ -124,6 +125,10 @@ private List<PTransformOverride> getOverrides() {
             PTransformOverride.of(
                 PTransformMatchers.splittableParDoMulti(),
                 new SplittableParDoOverrideFactory<>()))
+        .add(
+            PTransformOverride.of(
+                PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class),
+                new SplittableParDoViaKeyedWorkItems.OverrideFactory<>()))
         .build();
   }
 
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
index b3a6d1c0bf64..32e470f6029a 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
@@ -25,7 +25,8 @@
 import org.apache.beam.runners.apex.ApexRunner.CreateApexPCollectionView;
 import org.apache.beam.runners.apex.translation.operators.ApexProcessFnOperator;
 import org.apache.beam.runners.apex.translation.operators.ApexReadUnboundedInputOperator;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems;
 import org.apache.beam.runners.core.construction.PrimitiveCreate;
 import org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter;
 import org.apache.beam.sdk.Pipeline;
@@ -63,9 +64,9 @@ public class ApexPipelineTranslator implements Pipeline.PipelineVisitor {
   static {
     // register TransformTranslators
     registerTransformTranslator(ParDo.MultiOutput.class, new ParDoTranslator<>());
-    registerTransformTranslator(SplittableParDo.ProcessElements.class,
+    registerTransformTranslator(SplittableParDoViaKeyedWorkItems.ProcessElements.class,
         new ParDoTranslator.SplittableProcessElementsTranslator());
-    registerTransformTranslator(SplittableParDo.GBKIntoKeyedWorkItems.class,
+    registerTransformTranslator(GBKIntoKeyedWorkItems.class,
         new GBKIntoKeyedWorkItemsTranslator());
     registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
     registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
@@ -183,11 +184,11 @@ public void translate(
   }
 
   private static class GBKIntoKeyedWorkItemsTranslator<K, InputT>
-    implements TransformTranslator<SplittableParDo.GBKIntoKeyedWorkItems<K, InputT>> {
+    implements TransformTranslator<GBKIntoKeyedWorkItems<K, InputT>> {
 
     @Override
     public void translate(
-        SplittableParDo.GBKIntoKeyedWorkItems<K, InputT> transform, TranslationContext context) {
+        GBKIntoKeyedWorkItems<K, InputT> transform, TranslationContext context) {
       // https://issues.apache.org/jira/browse/BEAM-1850
       ApexProcessFnOperator<KV<K, InputT>> operator = ApexProcessFnOperator.toKeyedWorkItems(
           context.getPipelineOptions());
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
index 9133cb66f651..e46687a65858 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ParDoTranslator.java
@@ -30,7 +30,7 @@
 import java.util.Map.Entry;
 import org.apache.beam.runners.apex.ApexRunner;
 import org.apache.beam.runners.apex.translation.operators.ApexParDoOperator;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessElements;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -132,14 +132,13 @@ public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationC
     }
   }
 
-  static class SplittableProcessElementsTranslator<InputT, OutputT,
-      RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
-    implements TransformTranslator<SplittableParDo.ProcessElements<InputT, OutputT,
-      RestrictionT, TrackerT>> {
+  static class SplittableProcessElementsTranslator<
+          InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+      implements TransformTranslator<ProcessElements<InputT, OutputT, RestrictionT, TrackerT>> {
 
     @Override
     public void translate(
-        SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform,
+        ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform,
         TranslationContext context) {
 
       Map<TupleTag<?>, PValue> outputs = context.getOutputs();
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
index 7fee0d5fcdf4..809ca2a166c5 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexParDoOperator.java
@@ -55,7 +55,7 @@
 import org.apache.beam.runners.core.SideInputHandler;
 import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StateInternalsFactory;
 import org.apache.beam.runners.core.StateNamespace;
@@ -160,7 +160,7 @@ public ApexParDoOperator(
         TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
     this.currentKeyTimerInternals = new ApexTimerInternals<>(timerCoder);
 
-    if (doFn instanceof SplittableParDo.ProcessFn) {
+    if (doFn instanceof ProcessFn) {
       // we know that it is keyed on String
       Coder<?> keyCoder = StringUtf8Coder.of();
       this.currentKeyStateInternals = new StateInternalsProxy<>(
@@ -445,15 +445,15 @@ public TimerInternals timerInternals() {
     pushbackDoFnRunner =
         SimplePushbackSideInputDoFnRunner.create(doFnRunner, sideInputs, sideInputHandler);
 
-    if (doFn instanceof SplittableParDo.ProcessFn) {
+    if (doFn instanceof ProcessFn) {
 
       @SuppressWarnings("unchecked")
       StateInternalsFactory<String> stateInternalsFactory =
           (StateInternalsFactory<String>) this.currentKeyStateInternals.getFactory();
 
       @SuppressWarnings({ "rawtypes", "unchecked" })
-      SplittableParDo.ProcessFn<InputT, OutputT, Object, RestrictionTracker<Object>>
-        splittableDoFn = (SplittableParDo.ProcessFn) doFn;
+      ProcessFn<InputT, OutputT, Object, RestrictionTracker<Object>>
+        splittableDoFn = (ProcessFn) doFn;
       splittableDoFn.setStateInternalsFactory(stateInternalsFactory);
       TimerInternalsFactory<String> timerInternalsFactory = new TimerInternalsFactory<String>() {
          @Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ElementAndRestriction.java
similarity index 97%
rename from runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ElementAndRestriction.java
index 4a5d0c46168e..53a86b1d4f99 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestriction.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ElementAndRestriction.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.runners.core;
+package org.apache.beam.runners.core.construction;
 
 import com.google.auto.value.AutoValue;
 import org.apache.beam.sdk.annotations.Experimental;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ElementAndRestrictionCoder.java
similarity index 98%
rename from runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ElementAndRestrictionCoder.java
index 4440b856318b..5ff0aaead9eb 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ElementAndRestrictionCoder.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ElementAndRestrictionCoder.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.runners.core;
+package org.apache.beam.runners.core.construction;
 
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
new file mode 100644
index 000000000000..23ba66ab983d
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.List;
+import java.util.UUID;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
+
+/**
+ * A utility transform that executes a <a
+ * href="https://s.apache.org/splittable-do-fn">splittable</a> {@link DoFn} by expanding it into a
+ * network of simpler transforms:
+ *
+ * <ol>
+ * <li>Pair each element with an initial restriction
+ * <li>Split each restriction into sub-restrictions
+ * <li>Explode windows, since splitting within each window has to happen independently
+ * <li>Assign a unique key to each element/restriction pair
+ * <li>Process the keyed element/restriction pairs in a runner-specific way with the splittable
+ *   {@link DoFn}'s {@link DoFn.ProcessElement} method.
+ * </ol>
+ *
+ * <p>This transform is intended as a helper for internal use by runners when implementing {@code
+ * ParDo.of(splittable DoFn)}, but not for direct use by pipeline writers.
+ */
+@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
+public class SplittableParDo<InputT, OutputT, RestrictionT>
+    extends PTransform<PCollection<InputT>, PCollectionTuple> {
+  private final ParDo.MultiOutput<InputT, OutputT> parDo;
+
+  /**
+   * Creates the transform for the given original multi-output {@link ParDo}.
+   *
+   * @param parDo The splittable {@link ParDo} transform.
+   */
+  public SplittableParDo(ParDo.MultiOutput<InputT, OutputT> parDo) {
+    checkNotNull(parDo, "parDo must not be null");
+    this.parDo = parDo;
+    checkArgument(
+        DoFnSignatures.getSignature(parDo.getFn().getClass()).processElement().isSplittable(),
+        "fn must be a splittable DoFn");
+  }
+
+  @Override
+  public PCollectionTuple expand(PCollection<InputT> input) {
+    DoFn<InputT, OutputT> fn = parDo.getFn();
+    Coder<RestrictionT> restrictionCoder =
+        DoFnInvokers.invokerFor(fn)
+            .invokeGetRestrictionCoder(input.getPipeline().getCoderRegistry());
+    Coder<ElementAndRestriction<InputT, RestrictionT>> splitCoder =
+        ElementAndRestrictionCoder.of(input.getCoder(), restrictionCoder);
+
+    PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>> keyedRestrictions =
+        input
+            .apply(
+                "Pair with initial restriction",
+                ParDo.of(new PairWithRestrictionFn<InputT, OutputT, RestrictionT>(fn)))
+            .setCoder(splitCoder)
+            .apply("Split restriction", ParDo.of(new SplitRestrictionFn<InputT, RestrictionT>(fn)))
+            .setCoder(splitCoder)
+            // ProcessFn requires all input elements to be in a single window and have a single
+            // element per work item. This must precede the unique keying so each key has a single
+            // associated element.
+            .apply(
+                "Explode windows",
+                ParDo.of(new ExplodeWindowsFn<ElementAndRestriction<InputT, RestrictionT>>()))
+            .apply(
+                "Assign unique key",
+                WithKeys.of(new RandomUniqueKeyFn<ElementAndRestriction<InputT, RestrictionT>>()));
+
+    return keyedRestrictions.apply(
+        "ProcessKeyedElements",
+        new ProcessKeyedElements<>(
+            fn,
+            input.getCoder(),
+            restrictionCoder,
+            (WindowingStrategy<InputT, ?>) input.getWindowingStrategy(),
+            parDo.getSideInputs(),
+            parDo.getMainOutputTag(),
+            parDo.getAdditionalOutputTags()));
+  }
+
+  /**
+   * A {@link DoFn} that forces each of its outputs to be in a single window, by indicating to the
+   * runner that it observes the window of its input element, so the runner is forced to apply it to
+   * each input in a single window and thus its output is also in a single window.
+   */
+  private static class ExplodeWindowsFn<InputT> extends DoFn<InputT, InputT> {
+    @ProcessElement
+    public void process(ProcessContext c, BoundedWindow window) {
+      c.output(c.element());
+    }
+  }
+
+  /**
+   * Runner-specific primitive {@link PTransform} that invokes the {@link DoFn.ProcessElement}
+   * method for a splittable {@link DoFn} on each {@link ElementAndRestriction} of the input
+   * {@link PCollection} of {@link KV KVs} keyed with arbitrary but globally unique keys.
+   */
+  public static class ProcessKeyedElements<InputT, OutputT, RestrictionT>
+      extends PTransform<
+          PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>>, PCollectionTuple> {
+    private final DoFn<InputT, OutputT> fn;
+    private final Coder<InputT> elementCoder;
+    private final Coder<RestrictionT> restrictionCoder;
+    private final WindowingStrategy<InputT, ?> windowingStrategy;
+    private final List<PCollectionView<?>> sideInputs;
+    private final TupleTag<OutputT> mainOutputTag;
+    private final TupleTagList additionalOutputTags;
+
+    /**
+     * @param fn the splittable {@link DoFn}.
+     * @param windowingStrategy the {@link WindowingStrategy} of the input collection.
+     * @param sideInputs list of side inputs that should be available to the {@link DoFn}.
+     * @param mainOutputTag {@link TupleTag Tag} of the {@link DoFn DoFn's} main output.
+     * @param additionalOutputTags {@link TupleTagList Tags} of the {@link DoFn DoFn's} additional
+     *     outputs.
+     */
+    public ProcessKeyedElements(
+        DoFn<InputT, OutputT> fn,
+        Coder<InputT> elementCoder,
+        Coder<RestrictionT> restrictionCoder,
+        WindowingStrategy<InputT, ?> windowingStrategy,
+        List<PCollectionView<?>> sideInputs,
+        TupleTag<OutputT> mainOutputTag,
+        TupleTagList additionalOutputTags) {
+      this.fn = fn;
+      this.elementCoder = elementCoder;
+      this.restrictionCoder = restrictionCoder;
+      this.windowingStrategy = windowingStrategy;
+      this.sideInputs = sideInputs;
+      this.mainOutputTag = mainOutputTag;
+      this.additionalOutputTags = additionalOutputTags;
+    }
+
+    public DoFn<InputT, OutputT> getFn() {
+      return fn;
+    }
+
+    public Coder<InputT> getElementCoder() {
+      return elementCoder;
+    }
+
+    public Coder<RestrictionT> getRestrictionCoder() {
+      return restrictionCoder;
+    }
+
+    public WindowingStrategy<InputT, ?> getInputWindowingStrategy() {
+      return windowingStrategy;
+    }
+
+    public List<PCollectionView<?>> getSideInputs() {
+      return sideInputs;
+    }
+
+    public TupleTag<OutputT> getMainOutputTag() {
+      return mainOutputTag;
+    }
+
+    public TupleTagList getAdditionalOutputTags() {
+      return additionalOutputTags;
+    }
+
+    @Override
+    public PCollectionTuple expand(
+        PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>>
+            input) {
+      return createPrimitiveOutputFor(
+          input, fn, mainOutputTag, additionalOutputTags, windowingStrategy);
+    }
+
+    public static <OutputT> PCollectionTuple createPrimitiveOutputFor(
+        PCollection<?> input,
+        DoFn<?, OutputT> fn,
+        TupleTag<OutputT> mainOutputTag,
+        TupleTagList additionalOutputTags,
+        WindowingStrategy<?, ?> windowingStrategy) {
+      DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
+      PCollectionTuple outputs =
+          PCollectionTuple.ofPrimitiveOutputsInternal(
+              input.getPipeline(),
+              TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()),
+              windowingStrategy,
+              input.isBounded().and(signature.isBoundedPerElement()));
+
+      // Set output type descriptor similarly to how ParDo.MultiOutput does it.
+      outputs.get(mainOutputTag).setTypeDescriptor(fn.getOutputTypeDescriptor());
+
+      return outputs;
+    }
+  }
+
+  /**
+   * Assigns a random unique key to each element of the input collection, so that the output
+   * collection is effectively the same elements as input, but the per-key state and timers are now
+   * effectively per-element.
+   */
+  private static class RandomUniqueKeyFn<T> implements SerializableFunction<T, String> {
+    @Override
+    public String apply(T input) {
+      return UUID.randomUUID().toString();
+    }
+  }
+
+  /**
+   * Pairs each input element with its initial restriction using the given splittable {@link DoFn}.
+   */
+  private static class PairWithRestrictionFn<InputT, OutputT, RestrictionT>
+      extends DoFn<InputT, ElementAndRestriction<InputT, RestrictionT>> {
+    private DoFn<InputT, OutputT> fn;
+    private transient DoFnInvoker<InputT, OutputT> invoker;
+
+    PairWithRestrictionFn(DoFn<InputT, OutputT> fn) {
+      this.fn = fn;
+    }
+
+    @Setup
+    public void setup() {
+      invoker = DoFnInvokers.invokerFor(fn);
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext context) {
+      context.output(
+          ElementAndRestriction.of(
+              context.element(),
+              invoker.<RestrictionT>invokeGetInitialRestriction(context.element())));
+    }
+  }
+
+  /** Splits the restriction using the given {@link SplitRestriction} method. */
+  private static class SplitRestrictionFn<InputT, RestrictionT>
+      extends DoFn<
+          ElementAndRestriction<InputT, RestrictionT>,
+          ElementAndRestriction<InputT, RestrictionT>> {
+    private final DoFn<InputT, ?> splittableFn;
+    private transient DoFnInvoker<InputT, ?> invoker;
+
+    SplitRestrictionFn(DoFn<InputT, ?> splittableFn) {
+      this.splittableFn = splittableFn;
+    }
+
+    @Setup
+    public void setup() {
+      invoker = DoFnInvokers.invokerFor(splittableFn);
+    }
+
+    @ProcessElement
+    public void processElement(final ProcessContext c) {
+      final InputT element = c.element().element();
+      invoker.invokeSplitRestriction(
+          element,
+          c.element().restriction(),
+          new OutputReceiver<RestrictionT>() {
+            @Override
+            public void output(RestrictionT part) {
+              c.output(ElementAndRestriction.of(element, part));
+            }
+          });
+    }
+  }
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ElementAndRestrictionCoderTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ElementAndRestrictionCoderTest.java
similarity index 96%
rename from runners/core-java/src/test/java/org/apache/beam/runners/core/ElementAndRestrictionCoderTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ElementAndRestrictionCoderTest.java
index f5160469307d..051cbaa0008a 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ElementAndRestrictionCoderTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ElementAndRestrictionCoderTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.runners.core;
+package org.apache.beam.runners.core.construction;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -39,8 +39,7 @@
 import org.junit.runners.Parameterized.Parameter;
 
 /**
- * Tests for {@link ElementAndRestrictionCoder}. Parroted from {@link
- * org.apache.beam.sdk.coders.KvCoderTest}.
+ * Tests for {@link ElementAndRestrictionCoder}.
  */
 @RunWith(Parameterized.class)
 public class ElementAndRestrictionCoderTest<K, V> {
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
new file mode 100644
index 000000000000..6e4d6c458f04
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core.construction;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.BoundedPerElement;
+import org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
+import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link SplittableParDo}. */
+@RunWith(JUnit4.class)
+public class SplittableParDoTest {
+  // ----------------- Tests for whether the transform sets boundedness correctly --------------
+  private static class SomeRestriction
+      implements Serializable, HasDefaultTracker<SomeRestriction, SomeRestrictionTracker> {
+    @Override
+    public SomeRestrictionTracker newTracker() {
+      return new SomeRestrictionTracker(this);
+    }
+  }
+
+  private static class SomeRestrictionTracker implements RestrictionTracker<SomeRestriction> {
+    private final SomeRestriction someRestriction;
+
+    public SomeRestrictionTracker(SomeRestriction someRestriction) {
+      this.someRestriction = someRestriction;
+    }
+
+    @Override
+    public SomeRestriction currentRestriction() {
+      return someRestriction;
+    }
+
+    @Override
+    public SomeRestriction checkpoint() {
+      return someRestriction;
+    }
+
+    @Override
+    public void checkDone() {}
+  }
+
+  @BoundedPerElement
+  private static class BoundedFakeFn extends DoFn<Integer, String> {
+    @ProcessElement
+    public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
+
+    @GetInitialRestriction
+    public SomeRestriction getInitialRestriction(Integer element) {
+      return null;
+    }
+  }
+
+  @UnboundedPerElement
+  private static class UnboundedFakeFn extends DoFn<Integer, String> {
+    @ProcessElement
+    public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
+
+    @GetInitialRestriction
+    public SomeRestriction getInitialRestriction(Integer element) {
+      return null;
+    }
+  }
+
+  private static PCollection<Integer> makeUnboundedCollection(Pipeline pipeline) {
+    return pipeline
+        .apply("unbounded", Create.of(1, 2, 3))
+        .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
+  }
+
+  private static PCollection<Integer> makeBoundedCollection(Pipeline pipeline) {
+    return pipeline
+        .apply("bounded", Create.of(1, 2, 3))
+        .setIsBoundedInternal(PCollection.IsBounded.BOUNDED);
+  }
+
+  private static final TupleTag<String> MAIN_OUTPUT_TAG = new TupleTag<String>() {};
+
+  private ParDo.MultiOutput<Integer, String> makeParDo(DoFn<Integer, String> fn) {
+    return ParDo.of(fn).withOutputTags(MAIN_OUTPUT_TAG, TupleTagList.empty());
+  }
+
+  @Rule
+  public TestPipeline pipeline = TestPipeline.create();
+
+  @Test
+  public void testBoundednessForBoundedFn() {
+    pipeline.enableAbandonedNodeEnforcement(false);
+
+    DoFn<Integer, String> boundedFn = new BoundedFakeFn();
+    assertEquals(
+        "Applying a bounded SDF to a bounded collection produces a bounded collection",
+        PCollection.IsBounded.BOUNDED,
+        makeBoundedCollection(pipeline)
+            .apply("bounded to bounded", new SplittableParDo<>(makeParDo(boundedFn)))
+            .get(MAIN_OUTPUT_TAG)
+            .isBounded());
+    assertEquals(
+        "Applying a bounded SDF to an unbounded collection produces an unbounded collection",
+        PCollection.IsBounded.UNBOUNDED,
+        makeUnboundedCollection(pipeline)
+            .apply("bounded to unbounded", new SplittableParDo<>(makeParDo(boundedFn)))
+            .get(MAIN_OUTPUT_TAG)
+            .isBounded());
+  }
+
+  @Test
+  public void testBoundednessForUnboundedFn() {
+    pipeline.enableAbandonedNodeEnforcement(false);
+
+    DoFn<Integer, String> unboundedFn = new UnboundedFakeFn();
+    assertEquals(
+        "Applying an unbounded SDF to a bounded collection produces a bounded collection",
+        PCollection.IsBounded.UNBOUNDED,
+        makeBoundedCollection(pipeline)
+            .apply("unbounded to bounded", new SplittableParDo<>(makeParDo(unboundedFn)))
+            .get(MAIN_OUTPUT_TAG)
+            .isBounded());
+    assertEquals(
+        "Applying an unbounded SDF to an unbounded collection produces an unbounded collection",
+        PCollection.IsBounded.UNBOUNDED,
+        makeUnboundedCollection(pipeline)
+            .apply("unbounded to unbounded", new SplittableParDo<>(makeParDo(unboundedFn)))
+            .get(MAIN_OUTPUT_TAG)
+            .isBounded());
+  }
+}
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
index f3cca6fcd23b..71dfd11a496d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
@@ -20,7 +20,7 @@
 import java.util.Collection;
 import java.util.List;
 import org.apache.beam.runners.core.ExecutionContext.StepContext;
-import org.apache.beam.runners.core.SplittableParDo.ProcessFn;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
 import org.apache.beam.runners.core.StatefulDoFnRunner.CleanupTimer;
 import org.apache.beam.runners.core.StatefulDoFnRunner.StateCleaner;
 import org.apache.beam.sdk.options.PipelineOptions;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
index 61f413fd8cac..31e86bdd0dc5 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java
@@ -18,13 +18,13 @@
 package org.apache.beam.runners.core;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static org.apache.beam.runners.core.SplittableParDo.ProcessFn;
 
 import com.google.common.collect.Iterables;
 import java.util.Collection;
 import java.util.Collections;
 import org.apache.beam.runners.core.StateNamespaces.WindowNamespace;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.construction.ElementAndRestriction;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -32,7 +32,10 @@
 import org.apache.beam.sdk.values.PCollectionView;
 import org.joda.time.Instant;
 
-/** Runs a {@link ProcessFn} by constructing the appropriate contexts and passing them in. */
+/**
+ * Runs a {@link SplittableParDoViaKeyedWorkItems.ProcessFn} by constructing the appropriate
+ * contexts and passing them in.
+ */
 public class ProcessFnRunner<InputT, OutputT, RestrictionT>
     implements PushbackSideInputDoFnRunner<
         KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>, OutputT> {
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java
similarity index 53%
rename from runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
rename to runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java
index 6503fa20938d..b38e364c9856 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java
@@ -17,159 +17,48 @@
  */
 package org.apache.beam.runners.core;
 
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import java.util.List;
-import java.util.UUID;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import java.util.Map;
+import org.apache.beam.runners.core.construction.ElementAndRestriction;
+import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.runners.core.construction.SplittableParDo.ProcessKeyedElements;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.state.TimeDomain;
 import org.apache.beam.sdk.state.ValueState;
 import org.apache.beam.sdk.state.WatermarkHoldState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Instant;
 
 /**
- * A utility transform that executes a <a
- * href="https://s.apache.org/splittable-do-fn">splittable</a> {@link DoFn} by expanding it into a
- * network of simpler transforms:
- *
- * <ol>
- * <li>Pair each element with an initial restriction
- * <li>Split each restriction into sub-restrictions
- * <li>Assign a unique key to each element/restriction pair
- * <li>Group by key (so that work is partitioned by key and we can access state/timers)
- * <li>Process each keyed element/restriction pair with the splittable {@link DoFn}'s {@link
- *     DoFn.ProcessElement} method, using state and timers API.
- * </ol>
- *
- * <p>This transform is intended as a helper for internal use by runners when implementing {@code
- * ParDo.of(splittable DoFn)}, but not for direct use by pipeline writers.
+ * Utilities for implementing {@link ProcessKeyedElements} using {@link KeyedWorkItem} and
+ * runner-specific {@link StateInternals} and {@link TimerInternals}.
  */
-@Experimental(Experimental.Kind.SPLITTABLE_DO_FN)
-public class SplittableParDo<InputT, OutputT, RestrictionT>
-    extends PTransform<PCollection<InputT>, PCollectionTuple> {
-  private final ParDo.MultiOutput<InputT, OutputT> parDo;
-
-  /**
-   * Creates the transform for the given original multi-output {@link ParDo}.
-   *
-   * @param parDo The splittable {@link ParDo} transform.
-   */
-  public SplittableParDo(ParDo.MultiOutput<InputT, OutputT> parDo) {
-    checkNotNull(parDo, "parDo must not be null");
-    this.parDo = parDo;
-    checkArgument(
-        DoFnSignatures.getSignature(parDo.getFn().getClass()).processElement().isSplittable(),
-        "fn must be a splittable DoFn");
-  }
-
-  @Override
-  public PCollectionTuple expand(PCollection<InputT> input) {
-    return applyTyped(input);
-  }
-
-  private PCollectionTuple applyTyped(PCollection<InputT> input) {
-    DoFn<InputT, OutputT> fn = parDo.getFn();
-    Coder<RestrictionT> restrictionCoder =
-        DoFnInvokers.invokerFor(fn)
-            .invokeGetRestrictionCoder(input.getPipeline().getCoderRegistry());
-    PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>> keyedWorkItems =
-        applySplitIntoKeyedWorkItems(input, fn, restrictionCoder);
-    return keyedWorkItems.apply(
-        "Process",
-        new ProcessElements<>(
-            fn,
-            input.getCoder(),
-            restrictionCoder,
-            (WindowingStrategy<InputT, ?>) input.getWindowingStrategy(),
-            parDo.getSideInputs(),
-            parDo.getMainOutputTag(),
-            parDo.getAdditionalOutputTags()));
-  }
-
-  private static <InputT, OutputT, RestrictionT>
-      PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
-          applySplitIntoKeyedWorkItems(
-              PCollection<InputT> input,
-              DoFn<InputT, OutputT> fn,
-              Coder<RestrictionT> restrictionCoder) {
-    Coder<ElementAndRestriction<InputT, RestrictionT>> splitCoder =
-        ElementAndRestrictionCoder.of(input.getCoder(), restrictionCoder);
-
-    PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>> keyedWorkItems =
-        input
-            .apply(
-                "Pair with initial restriction",
-                ParDo.of(new PairWithRestrictionFn<InputT, OutputT, RestrictionT>(fn)))
-            .setCoder(splitCoder)
-            .apply("Split restriction", ParDo.of(new SplitRestrictionFn<InputT, RestrictionT>(fn)))
-            .setCoder(splitCoder)
-            // ProcessFn requires all input elements to be in a single window and have a single
-            // element per work item. This must precede the unique keying so each key has a single
-            // associated element.
-            .apply(
-                "Explode windows",
-                ParDo.of(new ExplodeWindowsFn<ElementAndRestriction<InputT, RestrictionT>>()))
-            .apply(
-                "Assign unique key",
-                WithKeys.of(new RandomUniqueKeyFn<ElementAndRestriction<InputT, RestrictionT>>()))
-            .apply(
-                "Group by key",
-                new GBKIntoKeyedWorkItems<String, ElementAndRestriction<InputT, RestrictionT>>())
-            .setCoder(
-                KeyedWorkItemCoder.of(
-                    StringUtf8Coder.of(),
-                    splitCoder,
-                    input.getWindowingStrategy().getWindowFn().windowCoder()));
-    checkArgument(
-        keyedWorkItems.getWindowingStrategy().getWindowFn() instanceof GlobalWindows,
-        "GBKIntoKeyedWorkItems must produce a globally windowed collection, "
-            + "but windowing strategy was: %s",
-        keyedWorkItems.getWindowingStrategy());
-    return keyedWorkItems;
-  }
-
-  /**
-   * A {@link DoFn} that forces each of its outputs to be in a single window, by indicating to the
-   * runner that it observes the window of its input element, so the runner is forced to apply it to
-   * each input in a single window and thus its output is also in a single window.
-   */
-  private static class ExplodeWindowsFn<InputT> extends DoFn<InputT, InputT> {
-    @ProcessElement
-    public void process(ProcessContext c, BoundedWindow window) {
-      c.output(c.element());
-    }
-  }
-
+public class SplittableParDoViaKeyedWorkItems {
   /**
    * Runner-specific primitive {@link GroupByKey GroupByKey-like} {@link PTransform} that produces
    * {@link KeyedWorkItem KeyedWorkItems} so that downstream transforms can access state and timers.
@@ -186,143 +75,106 @@ public PCollection<KeyedWorkItem<KeyT, InputT>> expand(PCollection<KV<KeyT, Inpu
     }
   }
 
+  /** Overrides a {@link ProcessKeyedElements} into {@link SplittableProcessViaKeyedWorkItems}. */
+  public static class OverrideFactory<InputT, OutputT, RestrictionT>
+      implements PTransformOverrideFactory<
+          PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>>, PCollectionTuple,
+      ProcessKeyedElements<InputT, OutputT, RestrictionT>> {
+    @Override
+    public PTransformReplacement<
+            PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>>, PCollectionTuple>
+        getReplacementTransform(
+            AppliedPTransform<
+                    PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>>,
+                    PCollectionTuple, ProcessKeyedElements<InputT, OutputT, RestrictionT>>
+                transform) {
+      return PTransformReplacement.of(
+          PTransformReplacements.getSingletonMainInput(transform),
+          new SplittableProcessViaKeyedWorkItems<>(transform.getTransform()));
+    }
+
+    @Override
+    public Map<PValue, ReplacementOutput> mapOutputs(
+        Map<TupleTag<?>, PValue> outputs, PCollectionTuple newOutput) {
+      return ReplacementOutputs.tagged(outputs, newOutput);
+    }
+  }
+
   /**
    * Runner-specific primitive {@link PTransform} that invokes the {@link DoFn.ProcessElement}
    * method for a splittable {@link DoFn}.
    */
-  public static class ProcessElements<
-          InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+  public static class SplittableProcessViaKeyedWorkItems<InputT, OutputT, RestrictionT>
       extends PTransform<
-          PCollection<? extends KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>,
-          PCollectionTuple> {
-    private final DoFn<InputT, OutputT> fn;
-    private final Coder<InputT> elementCoder;
-    private final Coder<RestrictionT> restrictionCoder;
-    private final WindowingStrategy<InputT, ?> windowingStrategy;
-    private final List<PCollectionView<?>> sideInputs;
-    private final TupleTag<OutputT> mainOutputTag;
-    private final TupleTagList additionalOutputTags;
-
-    /**
-     * @param fn the splittable {@link DoFn}.
-     * @param windowingStrategy the {@link WindowingStrategy} of the input collection.
-     * @param sideInputs list of side inputs that should be available to the {@link DoFn}.
-     * @param mainOutputTag {@link TupleTag Tag} of the {@link DoFn DoFn's} main output.
-     * @param additionalOutputTags {@link TupleTagList Tags} of the {@link DoFn DoFn's} additional
-     *     outputs.
-     */
-    public ProcessElements(
-        DoFn<InputT, OutputT> fn,
-        Coder<InputT> elementCoder,
-        Coder<RestrictionT> restrictionCoder,
-        WindowingStrategy<InputT, ?> windowingStrategy,
-        List<PCollectionView<?>> sideInputs,
-        TupleTag<OutputT> mainOutputTag,
-        TupleTagList additionalOutputTags) {
-      this.fn = fn;
-      this.elementCoder = elementCoder;
-      this.restrictionCoder = restrictionCoder;
-      this.windowingStrategy = windowingStrategy;
-      this.sideInputs = sideInputs;
-      this.mainOutputTag = mainOutputTag;
-      this.additionalOutputTags = additionalOutputTags;
-    }
+          PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>>, PCollectionTuple> {
+    private final ProcessKeyedElements<InputT, OutputT, RestrictionT> original;
 
-    public DoFn<InputT, OutputT> getFn() {
-      return fn;
+    public SplittableProcessViaKeyedWorkItems(
+        ProcessKeyedElements<InputT, OutputT, RestrictionT> original) {
+      this.original = original;
     }
 
-    public List<PCollectionView<?>> getSideInputs() {
-      return sideInputs;
+    @Override
+    public PCollectionTuple expand(
+        PCollection<KV<String, ElementAndRestriction<InputT, RestrictionT>>> input) {
+      return input
+          .apply(new GBKIntoKeyedWorkItems<String, ElementAndRestriction<InputT, RestrictionT>>())
+          .setCoder(
+              KeyedWorkItemCoder.of(
+                  StringUtf8Coder.of(),
+                  ((KvCoder<String, ElementAndRestriction<InputT, RestrictionT>>) input.getCoder())
+                      .getValueCoder(),
+                  input.getWindowingStrategy().getWindowFn().windowCoder()))
+          .apply(new ProcessElements<>(original));
     }
+  }
 
-    public TupleTag<OutputT> getMainOutputTag() {
-      return mainOutputTag;
-    }
+  /** A primitive transform wrapping around {@link ProcessFn}. */
+  public static class ProcessElements<
+          InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
+      extends PTransform<
+          PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>,
+          PCollectionTuple> {
+    private final ProcessKeyedElements<InputT, OutputT, RestrictionT> original;
 
-    public TupleTagList getAdditionalOutputTags() {
-      return additionalOutputTags;
+    public ProcessElements(ProcessKeyedElements<InputT, OutputT, RestrictionT> original) {
+      this.original = original;
     }
 
     public ProcessFn<InputT, OutputT, RestrictionT, TrackerT> newProcessFn(
         DoFn<InputT, OutputT> fn) {
-      return new SplittableParDo.ProcessFn<>(
-          fn, elementCoder, restrictionCoder, windowingStrategy);
-    }
-
-    @Override
-    public PCollectionTuple expand(
-        PCollection<? extends KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
-            input) {
-      DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass());
-      PCollectionTuple outputs =
-          PCollectionTuple.ofPrimitiveOutputsInternal(
-              input.getPipeline(),
-              TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()),
-              windowingStrategy,
-              input.isBounded().and(signature.isBoundedPerElement()));
-
-      // Set output type descriptor similarly to how ParDo.MultiOutput does it.
-      outputs.get(mainOutputTag).setTypeDescriptor(fn.getOutputTypeDescriptor());
-
-      return outputs;
+      return new ProcessFn<>(
+          fn,
+          original.getElementCoder(),
+          original.getRestrictionCoder(),
+          original.getInputWindowingStrategy());
     }
 
-    @Override
-    public <T> Coder<T> getDefaultOutputCoder(
-        PCollection<? extends KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
-            input,
-        PCollection<T> output)
-        throws CannotProvideCoderException {
-      // Similar logic to ParDo.MultiOutput.getOutputCoder.
-      @SuppressWarnings("unchecked")
-      KeyedWorkItemCoder<String, ElementAndRestriction<InputT, RestrictionT>> kwiCoder =
-          (KeyedWorkItemCoder) input.getCoder();
-      Coder<InputT> inputCoder =
-          ((ElementAndRestrictionCoder<InputT, RestrictionT>) kwiCoder.getElementCoder())
-              .getElementCoder();
-      return input
-          .getPipeline()
-          .getCoderRegistry()
-          .getCoder(output.getTypeDescriptor(), fn.getInputTypeDescriptor(), inputCoder);
+    public DoFn<InputT, OutputT> getFn() {
+      return original.getFn();
     }
-  }
 
-  /**
-   * Assigns a random unique key to each element of the input collection, so that the output
-   * collection is effectively the same elements as input, but the per-key state and timers are now
-   * effectively per-element.
-   */
-  private static class RandomUniqueKeyFn<T> implements SerializableFunction<T, String> {
-    @Override
-    public String apply(T input) {
-      return UUID.randomUUID().toString();
+    public List<PCollectionView<?>> getSideInputs() {
+      return original.getSideInputs();
     }
-  }
 
-  /**
-   * Pairs each input element with its initial restriction using the given splittable {@link DoFn}.
-   */
-  private static class PairWithRestrictionFn<InputT, OutputT, RestrictionT>
-      extends DoFn<InputT, ElementAndRestriction<InputT, RestrictionT>> {
-    private DoFn<InputT, OutputT> fn;
-    private transient DoFnInvoker<InputT, OutputT> invoker;
-
-    PairWithRestrictionFn(DoFn<InputT, OutputT> fn) {
-      this.fn = fn;
+    public TupleTag<OutputT> getMainOutputTag() {
+      return original.getMainOutputTag();
     }
 
-    @Setup
-    public void setup() {
-      invoker = DoFnInvokers.invokerFor(fn);
+    public TupleTagList getAdditionalOutputTags() {
+      return original.getAdditionalOutputTags();
     }
 
-    @ProcessElement
-    public void processElement(ProcessContext context) {
-      context.output(
-          ElementAndRestriction.of(
-              context.element(),
-              invoker.<RestrictionT>invokeGetInitialRestriction(context.element())));
+    @Override
+    public PCollectionTuple expand(
+        PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>> input) {
+      return ProcessKeyedElements.createPrimitiveOutputFor(
+          input,
+          original.getFn(),
+          original.getMainOutputTag(),
+          original.getAdditionalOutputTags(),
+          original.getInputWindowingStrategy());
     }
   }
 
@@ -331,10 +183,10 @@ public void processElement(ProcessContext context) {
    * by creating a tracker for the restriction and checkpointing/resuming processing later if
    * necessary.
    *
-   * <p>Takes {@link KeyedWorkItem} and assumes that the KeyedWorkItem contains a single element
-   * (or a single timer set by {@link ProcessFn itself}, in a single window. This is necessary
-   * because {@link ProcessFn} sets timers, and timers are namespaced to a single window and it
-   * should be the window of the input element.
+   * <p>Takes {@link KeyedWorkItem} and assumes that the KeyedWorkItem contains a single element (or
+   * a single timer set by {@link ProcessFn itself}, in a single window. This is necessary because
+   * {@link ProcessFn} sets timers, and timers are namespaced to a single window and it should be
+   * the window of the input element.
    *
    * <p>See also: https://issues.apache.org/jira/browse/BEAM-1983
    */
@@ -354,8 +206,7 @@ public static class ProcessFn<
      */
     private static final StateTag<WatermarkHoldState> watermarkHoldTag =
         StateTags.makeSystemTagInternal(
-            StateTags.<GlobalWindow>watermarkStateInternal(
-                "hold", TimestampCombiner.LATEST));
+            StateTags.<GlobalWindow>watermarkStateInternal("hold", TimestampCombiner.LATEST));
 
     /**
      * The state cell containing a copy of the element. Written during the first {@link
@@ -452,8 +303,7 @@ public void finishBundle(FinishBundleContext c) throws Exception {
     @ProcessElement
     public void processElement(final ProcessContext c) {
       String key = c.element().key();
-      StateInternals stateInternals =
-          stateInternalsFactory.stateInternalsForKey(key);
+      StateInternals stateInternals = stateInternalsFactory.stateInternalsForKey(key);
       TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key);
 
       // Initialize state (element and restriction) depending on whether this is the seed call.
@@ -477,8 +327,7 @@ public void processElement(final ProcessContext c) {
           stateInternals.state(stateNamespace, elementTag);
       ValueState<RestrictionT> restrictionState =
           stateInternals.state(stateNamespace, restrictionTag);
-      WatermarkHoldState holdState =
-          stateInternals.state(stateNamespace, watermarkHoldTag);
+      WatermarkHoldState holdState = stateInternals.state(stateNamespace, watermarkHoldTag);
 
       ElementAndRestriction<WindowedValue<InputT>, RestrictionT> elementAndRestriction;
       if (isSeedCall) {
@@ -565,38 +414,5 @@ private void throwUnsupportedOutput() {
         }
       };
     }
-
-  }
-
-  /** Splits the restriction using the given {@link DoFn.SplitRestriction} method. */
-  private static class SplitRestrictionFn<InputT, RestrictionT>
-      extends DoFn<
-          ElementAndRestriction<InputT, RestrictionT>,
-          ElementAndRestriction<InputT, RestrictionT>> {
-    private final DoFn<InputT, ?> splittableFn;
-    private transient DoFnInvoker<InputT, ?> invoker;
-
-    SplitRestrictionFn(DoFn<InputT, ?> splittableFn) {
-      this.splittableFn = splittableFn;
-    }
-
-    @Setup
-    public void setup() {
-      invoker = DoFnInvokers.invokerFor(splittableFn);
-    }
-
-    @ProcessElement
-    public void processElement(final ProcessContext c) {
-      final InputT element = c.element().element();
-      invoker.invokeSplitRestriction(
-          element,
-          c.element().restriction(),
-          new OutputReceiver<RestrictionT>() {
-            @Override
-            public void output(RestrictionT part) {
-              c.output(ElementAndRestriction.of(element, part));
-            }
-          });
-    }
   }
 }
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java
similarity index 82%
rename from runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
rename to runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java
index be4cf08b662d..d2424318eea5 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java
@@ -34,19 +34,15 @@
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.concurrent.Executors;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
+import org.apache.beam.runners.core.construction.ElementAndRestriction;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.BoundedPerElement;
-import org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement;
 import org.apache.beam.sdk.transforms.DoFnTester;
-import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
 import org.apache.beam.sdk.transforms.splittabledofn.OffsetRange;
 import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
@@ -57,11 +53,9 @@
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.ValueInSingleWindow;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.joda.time.Duration;
@@ -71,9 +65,9 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
-/** Tests for {@link SplittableParDo}. */
+/** Tests for {@link SplittableParDoViaKeyedWorkItems.ProcessFn}. */
 @RunWith(JUnit4.class)
-public class SplittableParDoTest {
+public class SplittableParDoProcessFnTest {
   private static final int MAX_OUTPUTS_PER_BUNDLE = 10000;
   private static final Duration MAX_BUNDLE_DURATION = Duration.standardSeconds(5);
 
@@ -107,96 +101,12 @@ public SomeRestriction checkpoint() {
     public void checkDone() {}
   }
 
-  @BoundedPerElement
-  private static class BoundedFakeFn extends DoFn<Integer, String> {
-    @ProcessElement
-    public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
-
-    @GetInitialRestriction
-    public SomeRestriction getInitialRestriction(Integer element) {
-      return null;
-    }
-  }
-
-  @UnboundedPerElement
-  private static class UnboundedFakeFn extends DoFn<Integer, String> {
-    @ProcessElement
-    public void processElement(ProcessContext context, SomeRestrictionTracker tracker) {}
-
-    @GetInitialRestriction
-    public SomeRestriction getInitialRestriction(Integer element) {
-      return null;
-    }
-  }
-
-  private static PCollection<Integer> makeUnboundedCollection(Pipeline pipeline) {
-    return pipeline
-        .apply("unbounded", Create.of(1, 2, 3))
-        .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
-  }
-
-  private static PCollection<Integer> makeBoundedCollection(Pipeline pipeline) {
-    return pipeline
-        .apply("bounded", Create.of(1, 2, 3))
-        .setIsBoundedInternal(PCollection.IsBounded.BOUNDED);
-  }
-
-  private static final TupleTag<String> MAIN_OUTPUT_TAG = new TupleTag<String>() {};
-
-  private ParDo.MultiOutput<Integer, String> makeParDo(DoFn<Integer, String> fn) {
-    return ParDo.of(fn).withOutputTags(MAIN_OUTPUT_TAG, TupleTagList.empty());
-  }
-
   @Rule
   public TestPipeline pipeline = TestPipeline.create();
 
-  @Test
-  public void testBoundednessForBoundedFn() {
-    pipeline.enableAbandonedNodeEnforcement(false);
-
-    DoFn<Integer, String> boundedFn = new BoundedFakeFn();
-    assertEquals(
-        "Applying a bounded SDF to a bounded collection produces a bounded collection",
-        PCollection.IsBounded.BOUNDED,
-        makeBoundedCollection(pipeline)
-            .apply("bounded to bounded", new SplittableParDo<>(makeParDo(boundedFn)))
-            .get(MAIN_OUTPUT_TAG)
-            .isBounded());
-    assertEquals(
-        "Applying a bounded SDF to an unbounded collection produces an unbounded collection",
-        PCollection.IsBounded.UNBOUNDED,
-        makeUnboundedCollection(pipeline)
-            .apply("bounded to unbounded", new SplittableParDo<>(makeParDo(boundedFn)))
-            .get(MAIN_OUTPUT_TAG)
-            .isBounded());
-  }
-
-  @Test
-  public void testBoundednessForUnboundedFn() {
-    pipeline.enableAbandonedNodeEnforcement(false);
-
-    DoFn<Integer, String> unboundedFn = new UnboundedFakeFn();
-    assertEquals(
-        "Applying an unbounded SDF to a bounded collection produces a bounded collection",
-        PCollection.IsBounded.UNBOUNDED,
-        makeBoundedCollection(pipeline)
-            .apply("unbounded to bounded", new SplittableParDo<>(makeParDo(unboundedFn)))
-            .get(MAIN_OUTPUT_TAG)
-            .isBounded());
-    assertEquals(
-        "Applying an unbounded SDF to an unbounded collection produces an unbounded collection",
-        PCollection.IsBounded.UNBOUNDED,
-        makeUnboundedCollection(pipeline)
-            .apply("unbounded to unbounded", new SplittableParDo<>(makeParDo(unboundedFn)))
-            .get(MAIN_OUTPUT_TAG)
-            .isBounded());
-  }
-
-  // ------------------------------- Tests for ProcessFn ---------------------------------
-
   /**
-   * A helper for testing {@link SplittableParDo.ProcessFn} on 1 element (but possibly over multiple
-   * {@link DoFn.ProcessElement} calls).
+   * A helper for testing {@link ProcessFn} on 1 element (but
+   * possibly over multiple {@link DoFn.ProcessElement} calls).
    */
   private static class ProcessFnTester<
           InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
@@ -221,8 +131,8 @@ private static class ProcessFnTester<
       // encode IntervalWindow's because that's what all tests here use.
       WindowingStrategy<InputT, BoundedWindow> windowingStrategy =
           (WindowingStrategy) WindowingStrategy.of(FixedWindows.of(Duration.standardSeconds(1)));
-      final SplittableParDo.ProcessFn<InputT, OutputT, RestrictionT, TrackerT> processFn =
-          new SplittableParDo.ProcessFn<>(
+      final ProcessFn<InputT, OutputT, RestrictionT, TrackerT> processFn =
+          new ProcessFn<>(
               fn, inputCoder, restrictionCoder, windowingStrategy);
       this.tester = DoFnTester.of(processFn);
       this.timerInternals = new InMemoryTimerInternals();
@@ -248,7 +158,6 @@ public TimerInternals timerInternalsForKey(String key) {
               tester.getPipelineOptions(),
               new OutputWindowedValueToDoFnTester<>(tester),
               new SideInputReader() {
-                @Nullable
                 @Override
                 public <T> T get(PCollectionView<T> view, BoundedWindow window) {
                   throw new NoSuchElementException();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
index 64eecc852afd..3fefe206b2df 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGBKIntoKeyedWorkItemsOverrideFactory.java
@@ -18,17 +18,14 @@
 package org.apache.beam.runners.direct;
 
 import org.apache.beam.runners.core.KeyedWorkItem;
-import org.apache.beam.runners.core.SplittableParDo.GBKIntoKeyedWorkItems;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 
-/**
- * Provides an implementation of {@link SplittableParDo.GBKIntoKeyedWorkItems} for the Direct
- * Runner.
- */
+/** Provides an implementation of {@link GBKIntoKeyedWorkItems} for the Direct Runner. */
 class DirectGBKIntoKeyedWorkItemsOverrideFactory<KeyT, InputT>
     extends SingleInputOutputOverrideFactory<
         PCollection<KV<KeyT, InputT>>, PCollection<KeyedWorkItem<KeyT, InputT>>,
@@ -39,7 +36,7 @@ class DirectGBKIntoKeyedWorkItemsOverrideFactory<KeyT, InputT>
       getReplacementTransform(
           AppliedPTransform<
                   PCollection<KV<KeyT, InputT>>, PCollection<KeyedWorkItem<KeyT, InputT>>,
-                  GBKIntoKeyedWorkItems<KeyT, InputT>>
+              GBKIntoKeyedWorkItems<KeyT, InputT>>
               transform) {
     return PTransformReplacement.of(
         PTransformReplacements.getSingletonMainInput(transform),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index 181896f3b3fc..69dea8fa4cb6 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -27,8 +27,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.runners.core.SplittableParDo.GBKIntoKeyedWorkItems;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
 import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
 import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory;
 import org.apache.beam.sdk.Pipeline;
@@ -244,7 +245,12 @@ private List<PTransformOverride> defaultTransformOverrides() {
                 PTransformMatchers.stateOrTimerParDoMulti(), new ParDoMultiOverrideFactory()))
         .add(
             PTransformOverride.of(
-                PTransformMatchers.classEqualTo(GBKIntoKeyedWorkItems.class),
+                PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class),
+                new SplittableParDoViaKeyedWorkItems.OverrideFactory()))
+        .add(
+            PTransformOverride.of(
+                PTransformMatchers.classEqualTo(
+                    SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems.class),
                 new DirectGBKIntoKeyedWorkItemsOverrideFactory())) /* Returns a GBKO */
         .add(
             PTransformOverride.of(
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
index f9b6eba215f2..347f313af85a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
@@ -23,7 +23,7 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
 import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
 import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
@@ -48,7 +48,7 @@ class KeyedPValueTrackingVisitor implements PipelineVisitor {
 
   private static final Set<Class<? extends PTransform>> PRODUCES_KEYED_OUTPUTS =
       ImmutableSet.of(
-          SplittableParDo.GBKIntoKeyedWorkItems.class,
+          SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems.class,
           DirectGroupByKeyOnly.class,
           DirectGroupAlsoByWindow.class);
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
index 89903da1ce00..be433db1ca00 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
@@ -23,9 +23,9 @@
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItemCoder;
 import org.apache.beam.runners.core.KeyedWorkItems;
-import org.apache.beam.runners.core.SplittableParDo;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index f490b0ba49d0..dc85d87bc93f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -24,18 +24,18 @@
 import java.util.concurrent.Executors;
 import org.apache.beam.runners.core.DoFnRunners;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.ElementAndRestriction;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.OutputAndTimeBoundedSplittableProcessElementInvoker;
 import org.apache.beam.runners.core.OutputWindowedValue;
 import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
 import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
-import org.apache.beam.runners.core.SplittableParDo;
-import org.apache.beam.runners.core.SplittableParDo.ProcessFn;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessElements;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StateInternalsFactory;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.TimerInternalsFactory;
+import org.apache.beam.runners.core.construction.ElementAndRestriction;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -88,12 +88,11 @@ public void cleanup() throws Exception {
       createEvaluator(
           AppliedPTransform<
                   PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>,
-                  PCollectionTuple,
-                  SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>>
+                  PCollectionTuple, ProcessElements<InputT, OutputT, RestrictionT, TrackerT>>
               application,
           CommittedBundle<InputT> inputBundle)
           throws Exception {
-    final SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform =
+    final ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform =
         application.getTransform();
 
     ProcessFn<InputT, OutputT, RestrictionT, TrackerT> processFn =
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
index d0e622d7f934..718cca2f5ddb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
@@ -25,7 +25,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
 import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
 import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
 import org.apache.beam.runners.direct.ParDoMultiOverrideFactory.StatefulParDo;
@@ -66,7 +66,7 @@ public static TransformEvaluatorRegistry defaultRegistry(EvaluationContext ctxt)
                 new TestStreamEvaluatorFactory(ctxt))
             // Runner-specific primitive used in expansion of SplittableParDo
             .put(
-                SplittableParDo.ProcessElements.class,
+                SplittableParDoViaKeyedWorkItems.ProcessElements.class,
                 new SplittableProcessElementsEvaluatorFactory<>(ctxt))
             .build();
     return new TransformEvaluatorRegistry(primitives);
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
index 35d1bcd19f68..53a1fa160ad1 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
@@ -20,11 +20,12 @@
 import com.google.common.collect.ImmutableList;
 import java.util.List;
 import java.util.Map;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
 import org.apache.beam.runners.core.construction.PTransformMatchers;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
 import org.apache.beam.runners.core.construction.ReplacementOutputs;
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
+import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.runners.core.construction.UnconsumedReads;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -78,6 +79,10 @@ public void translate(Pipeline pipeline) {
                 PTransformOverride.of(
                     PTransformMatchers.splittableParDoMulti(),
                     new SplittableParDoOverrideFactory()))
+            .add(
+                PTransformOverride.of(
+                    PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class),
+                    new SplittableParDoViaKeyedWorkItems.OverrideFactory()))
             .add(
                 PTransformOverride.of(
                     PTransformMatchers.classEqualTo(View.AsIterable.class),
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index 9a9320564739..00e9934aacbf 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -30,10 +30,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import org.apache.beam.runners.core.ElementAndRestriction;
 import org.apache.beam.runners.core.KeyedWorkItem;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
 import org.apache.beam.runners.core.SystemReduceFn;
+import org.apache.beam.runners.core.construction.ElementAndRestriction;
 import org.apache.beam.runners.flink.translation.functions.FlinkAssignWindows;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.DoFnOperator;
@@ -119,9 +119,11 @@ class FlinkStreamingTransformTranslators {
 
     TRANSLATORS.put(ParDo.MultiOutput.class, new ParDoStreamingTranslator());
     TRANSLATORS.put(
-        SplittableParDo.ProcessElements.class, new SplittableProcessElementsStreamingTranslator());
+        SplittableParDoViaKeyedWorkItems.ProcessElements.class,
+        new SplittableProcessElementsStreamingTranslator());
     TRANSLATORS.put(
-        SplittableParDo.GBKIntoKeyedWorkItems.class, new GBKIntoKeyedWorkItemsTranslator());
+        SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems.class,
+        new GBKIntoKeyedWorkItemsTranslator());
 
 
     TRANSLATORS.put(Window.Assign.class, new WindowAssignTranslator());
@@ -329,7 +331,8 @@ public RawUnionValue map(T o) throws Exception {
   }
 
   /**
-   * Helper for translating {@link ParDo.MultiOutput} and {@link SplittableParDo.ProcessElements}.
+   * Helper for translating {@link ParDo.MultiOutput} and {@link
+   * SplittableParDoViaKeyedWorkItems.ProcessElements}.
    */
   static class ParDoTranslationHelper {
 
@@ -382,7 +385,7 @@ static <InputT, OutputT> void translateParDo(
         keyCoder = ((KvCoder) input.getCoder()).getKeyCoder();
         inputDataStream = inputDataStream.keyBy(new KvToByteBufferKeySelector(keyCoder));
         stateful = true;
-      } else if (doFn instanceof SplittableParDo.ProcessFn) {
+      } else if (doFn instanceof SplittableParDoViaKeyedWorkItems.ProcessFn) {
         // we know that it is keyed on String
         keyCoder = StringUtf8Coder.of();
         stateful = true;
@@ -578,19 +581,19 @@ public DoFnOperator<InputT, OutputT, RawUnionValue> createDoFnOperator(
   private static class SplittableProcessElementsStreamingTranslator<
       InputT, OutputT, RestrictionT, TrackerT extends RestrictionTracker<RestrictionT>>
       extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
-      SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>> {
+      SplittableParDoViaKeyedWorkItems.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>> {
 
     @Override
     public void translateNode(
-        SplittableParDo.ProcessElements<InputT, OutputT, RestrictionT, TrackerT> transform,
+        SplittableParDoViaKeyedWorkItems.ProcessElements<InputT, OutputT, RestrictionT, TrackerT>
+            transform,
         FlinkStreamingTranslationContext context) {
 
       ParDoTranslationHelper.translateParDo(
           transform.getName(),
           transform.newProcessFn(transform.getFn()),
           context.getCurrentTransform().getFullName(),
-          (PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>)
-              context.getInput(transform),
+          context.getInput(transform),
           transform.getSideInputs(),
           context.getOutputs(transform),
           transform.getMainOutputTag(),
@@ -930,18 +933,18 @@ public void translateNode(
 
   private static class GBKIntoKeyedWorkItemsTranslator<K, InputT>
       extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
-      SplittableParDo.GBKIntoKeyedWorkItems<K, InputT>> {
+      SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<K, InputT>> {
 
     @Override
     boolean canTranslate(
-        SplittableParDo.GBKIntoKeyedWorkItems<K, InputT> transform,
+        SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<K, InputT> transform,
         FlinkStreamingTranslationContext context) {
       return true;
     }
 
     @Override
     public void translateNode(
-        SplittableParDo.GBKIntoKeyedWorkItems<K, InputT> transform,
+        SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems<K, InputT> transform,
         FlinkStreamingTranslationContext context) {
 
       PCollection<KV<K, InputT>> input = context.getInput(transform);
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
index 968fc0a540a7..4ac2ff522138 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java
@@ -26,16 +26,16 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
-import org.apache.beam.runners.core.ElementAndRestriction;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItems;
 import org.apache.beam.runners.core.OutputAndTimeBoundedSplittableProcessElementInvoker;
 import org.apache.beam.runners.core.OutputWindowedValue;
-import org.apache.beam.runners.core.SplittableParDo;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StateInternalsFactory;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.TimerInternalsFactory;
+import org.apache.beam.runners.core.construction.ElementAndRestriction;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -94,7 +94,7 @@ public SplittableDoFnOperator(
   public void open() throws Exception {
     super.open();
 
-    checkState(doFn instanceof SplittableParDo.ProcessFn);
+    checkState(doFn instanceof ProcessFn);
 
     StateInternalsFactory<String> stateInternalsFactory = new StateInternalsFactory<String>() {
       @Override
@@ -114,9 +114,9 @@ public TimerInternals timerInternalsForKey(String key) {
 
     executorService = Executors.newSingleThreadScheduledExecutor(Executors.defaultThreadFactory());
 
-    ((SplittableParDo.ProcessFn) doFn).setStateInternalsFactory(stateInternalsFactory);
-    ((SplittableParDo.ProcessFn) doFn).setTimerInternalsFactory(timerInternalsFactory);
-    ((SplittableParDo.ProcessFn) doFn).setProcessElementInvoker(
+    ((ProcessFn) doFn).setStateInternalsFactory(stateInternalsFactory);
+    ((ProcessFn) doFn).setTimerInternalsFactory(timerInternalsFactory);
+    ((ProcessFn) doFn).setProcessElementInvoker(
         new OutputAndTimeBoundedSplittableProcessElementInvoker<>(
             doFn,
             serializedOptions.getPipelineOptions(),

From 54808be14ff2e86088807adc3c0dc0965c8a6f03 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 18 May 2017 18:43:31 -0400
Subject: [PATCH 258/387] [BEAM-2318] Ensure that tables that don't exist are
 not created

We use the same test table name in two tests: one that expects the table not to exist, another that
creates the table. Obviously, the correctness of this will depend on the order in which unit tests
are executed.

Fix the flake by using different and more explicit test names.
---
 .../test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
index dbeab04c9731..1cdfc7f0096b 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
@@ -282,7 +282,7 @@ public void testWriting() throws Exception {
     /** Tests that when writing to a non-existent table, the write fails. */
     @Test
     public void testWritingFailsTableDoesNotExist() throws Exception {
-        final String table = "TEST-TABLE";
+        final String table = "TEST-TABLE-DOES-NOT-EXIST";
 
         PCollection<KV<byte[], Iterable<Mutation>>> emptyInput =
                 p.apply(Create.empty(HBaseIO.WRITE_CODER));
@@ -298,7 +298,7 @@ public void testWritingFailsTableDoesNotExist() throws Exception {
     /** Tests that when writing an element fails, the write fails. */
     @Test
     public void testWritingFailsBadElement() throws Exception {
-        final String table = "TEST-TABLE";
+        final String table = "TEST-TABLE-BAD-ELEMENT";
         final String key = "KEY";
         createTable(table);
 

From 1da8da79dcc623c53b35d97419566b736447f6b6 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 18 May 2017 16:19:29 -0400
Subject: [PATCH 259/387] [BEAM-2391] Clone Scan in HBaseReader

---
 .../org/apache/beam/sdk/io/hbase/HBaseIO.java   |  5 +++--
 .../apache/beam/sdk/io/hbase/HBaseIOTest.java   | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
index 3c42da91f27a..849873c05970 100644
--- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
+++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java
@@ -487,8 +487,9 @@ public boolean start() throws IOException {
             connection = ConnectionFactory.createConnection(configuration);
             TableName tableName = TableName.valueOf(tableId);
             Table table = connection.getTable(tableName);
-            Scan scan = source.read.serializableScan.get();
-            scanner = table.getScanner(scan);
+            // [BEAM-2319] We have to clone the Scan because the underlying scanner may mutate it.
+            Scan scanClone = new Scan(source.read.serializableScan.get());
+            scanner = table.getScanner(scanClone);
             iter = scanner.iterator();
             return advance();
         }
diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
index 1cdfc7f0096b..4a067895b22c 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
@@ -33,6 +33,7 @@
 import org.apache.beam.sdk.io.range.ByteKey;
 import org.apache.beam.sdk.io.range.ByteKeyRange;
 import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.SourceTestUtils;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
@@ -205,6 +206,22 @@ public void testReadingWithSplits() throws Exception {
         assertSourcesEqualReferenceSource(source, splits, null /* options */);
     }
 
+    /** Tests that a {@link HBaseSource} can be read twice, verifying its immutability. */
+    @Test
+    public void testReadingSourceTwice() throws Exception {
+        final String table = "TEST-READING-TWICE";
+        final int numRows = 10;
+
+        // Set up test table data and sample row keys for size estimation and splitting.
+        createTable(table);
+        writeData(table, numRows);
+
+        HBaseIO.Read read = HBaseIO.read().withConfiguration(conf).withTableId(table);
+        HBaseSource source = new HBaseSource(read, null /* estimatedSizeBytes */);
+        assertThat(SourceTestUtils.readFromSource(source, null), hasSize(numRows));
+        // second read.
+        assertThat(SourceTestUtils.readFromSource(source, null), hasSize(numRows));
+    }
 
     /** Tests reading all rows using a filter. */
     @Test

From f969ea443c90345cf8c33f953c399e46223e8d03 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Mon, 15 May 2017 13:39:12 -0700
Subject: [PATCH 260/387] [BEAM-2252] Use pluginManagement instead of
 redefining maven-shade-plugin to inherit build order from root pom.xml

---
 runners/direct-java/pom.xml                | 175 ++++++++++-----------
 runners/google-cloud-dataflow-java/pom.xml | 121 +++++++-------
 sdks/java/core/pom.xml                     | 165 ++++++++++---------
 sdks/java/io/google-cloud-platform/pom.xml |  33 ++--
 4 files changed, 237 insertions(+), 257 deletions(-)

diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index c581113120dd..857dcf42fd7b 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -40,6 +40,92 @@
       </resource>
     </resources>
 
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>bundle-and-repackage</id>
+              <phase>package</phase>
+              <goals>
+                <goal>shade</goal>
+              </goals>
+              <configuration>
+                <shadeTestJar>true</shadeTestJar>
+                <artifactSet>
+                  <includes>
+                    <include>com.google.guava:guava</include>
+                    <include>com.google.protobuf:protobuf-java</include>
+                    <include>org.apache.beam:beam-runners-core-construction-java</include>
+                    <include>org.apache.beam:beam-runners-core-java</include>
+                    <include>org.apache.beam:beam-sdks-common-runner-api</include>
+                    <include>com.google.code.findbugs:jsr305</include>
+                  </includes>
+                </artifactSet>
+                <filters>
+                  <filter>
+                    <artifact>*:*</artifact>
+                    <excludes>
+                      <exclude>META-INF/*.SF</exclude>
+                      <exclude>META-INF/*.DSA</exclude>
+                      <exclude>META-INF/*.RSA</exclude>
+                    </excludes>
+                  </filter>
+                </filters>
+                <relocations>
+                  <relocation>
+                    <pattern>org.apache.beam.runners.core</pattern>
+                    <shadedPattern>
+                      org.apache.beam.runners.direct.repackaged.runners.core
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.beam.sdk.common</pattern>
+                    <shadedPattern>
+                      org.apache.beam.runners.direct.repackaged.sdk.common
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>com.google.common</pattern>
+                    <excludes>
+                      <!-- com.google.common is too generic, need to exclude guava-testlib -->
+                      <exclude>com.google.common.**.testing.*</exclude>
+                    </excludes>
+                    <shadedPattern>
+                      org.apache.beam.runners.direct.repackaged.com.google.common
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>com.google.protobuf</pattern>
+                    <shadedPattern>
+                      org.apache.beam.runners.direct.repackaged.com.google.protobuf
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>com.google.thirdparty</pattern>
+                    <shadedPattern>
+                      org.apache.beam.runners.direct.repackaged.com.google.thirdparty
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>javax.annotation</pattern>
+                    <shadedPattern>
+                      org.apache.beam.runners.direct.repackaged.javax.annotation
+                    </shadedPattern>
+                  </relocation>
+                </relocations>
+                <transformers>
+                  <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                </transformers>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -77,95 +163,6 @@
         </executions>
       </plugin>
 
-      <!-- Ensure that the Maven jar plugin runs before the Maven
-        shade plugin by listing the plugin higher within the file. -->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-      </plugin>
-
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>bundle-and-repackage</id>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <shadeTestJar>true</shadeTestJar>
-              <artifactSet>
-                <includes>
-                  <include>com.google.guava:guava</include>
-                  <include>com.google.protobuf:protobuf-java</include>
-                  <include>org.apache.beam:beam-runners-core-construction-java</include>
-                  <include>org.apache.beam:beam-runners-core-java</include>
-                  <include>org.apache.beam:beam-sdks-common-runner-api</include>
-                  <include>com.google.code.findbugs:jsr305</include>
-                </includes>
-              </artifactSet>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/*.SF</exclude>
-                    <exclude>META-INF/*.DSA</exclude>
-                    <exclude>META-INF/*.RSA</exclude>
-                  </excludes>
-                </filter>
-              </filters>
-              <relocations>
-                <relocation>
-                  <pattern>org.apache.beam.runners.core</pattern>
-                  <shadedPattern>
-                    org.apache.beam.runners.direct.repackaged.runners.core
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>org.apache.beam.sdk.common</pattern>
-                  <shadedPattern>
-                    org.apache.beam.runners.direct.repackaged.sdk.common
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.google.common</pattern>
-                  <excludes>
-                    <!-- com.google.common is too generic, need to exclude guava-testlib -->
-                    <exclude>com.google.common.**.testing.*</exclude>
-                  </excludes>
-                  <shadedPattern>
-                    org.apache.beam.runners.direct.repackaged.com.google.common
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.google.protobuf</pattern>
-                  <shadedPattern>
-                    org.apache.beam.runners.direct.repackaged.com.google.protobuf
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.google.thirdparty</pattern>
-                  <shadedPattern>
-                    org.apache.beam.runners.direct.repackaged.com.google.thirdparty
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>javax.annotation</pattern>
-                  <shadedPattern>
-                    org.apache.beam.runners.direct.repackaged.javax.annotation
-                  </shadedPattern>
-                </relocation>
-              </relocations>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
       <!-- Coverage analysis for unit tests. -->
       <plugin>
         <groupId>org.jacoco</groupId>
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 9577ceb07cc3..c09b7f407a5b 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -142,6 +142,63 @@
             <skip>true</skip>
           </configuration>
         </plugin>
+
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>bundle-and-repackage</id>
+              <phase>package</phase>
+              <goals>
+                <goal>shade</goal>
+              </goals>
+              <configuration>
+                <shadeTestJar>true</shadeTestJar>
+                <artifactSet>
+                  <includes>
+                    <include>com.google.guava:guava</include>
+                    <include>org.apache.beam:beam-runners-core-construction-java</include>
+                  </includes>
+                </artifactSet>
+                <filters>
+                  <filter>
+                    <artifact>*:*</artifact>
+                    <excludes>
+                      <exclude>META-INF/*.SF</exclude>
+                      <exclude>META-INF/*.DSA</exclude>
+                      <exclude>META-INF/*.RSA</exclude>
+                    </excludes>
+                  </filter>
+                </filters>
+                <relocations>
+                  <!-- TODO: Once ready, change the following pattern to 'com'
+                       only, exclude 'org.apache.beam.**', and remove
+                       the second relocation. -->
+                  <relocation>
+                    <pattern>com.google.common</pattern>
+                    <excludes>
+                      <!-- com.google.common is too generic, need to exclude guava-testlib -->
+                      <exclude>com.google.common.**.testing.*</exclude>
+                    </excludes>
+                    <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.common</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>com.google.thirdparty</pattern>
+                    <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.thirdparty</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.beam.runners.core</pattern>
+                    <shadedPattern>org.apache.beam.runners.dataflow.repackaged.org.apache.beam.runners.core</shadedPattern>
+                  </relocation>
+                </relocations>
+                <transformers>
+                  <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                </transformers>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
       </plugins>
     </pluginManagement>
 
@@ -173,70 +230,6 @@
         </executions>
       </plugin>
 
-      <!-- Ensure that the Maven jar plugin runs before the Maven
-        shade plugin by listing the plugin higher within the file. -->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-      </plugin>
-
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>bundle-and-repackage</id>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <shadeTestJar>true</shadeTestJar>
-              <artifactSet>
-                <includes>
-                  <include>com.google.guava:guava</include>
-                  <include>org.apache.beam:beam-runners-core-construction-java</include>
-                </includes>
-              </artifactSet>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/*.SF</exclude>
-                    <exclude>META-INF/*.DSA</exclude>
-                    <exclude>META-INF/*.RSA</exclude>
-                  </excludes>
-                </filter>
-              </filters>
-              <relocations>
-                <!-- TODO: Once ready, change the following pattern to 'com'
-                     only, exclude 'org.apache.beam.**', and remove
-                     the second relocation. -->
-                <relocation>
-                  <pattern>com.google.common</pattern>
-                  <excludes>
-                    <!-- com.google.common is too generic, need to exclude guava-testlib -->
-                    <exclude>com.google.common.**.testing.*</exclude>
-                  </excludes>
-                  <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.common</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.google.thirdparty</pattern>
-                  <shadedPattern>org.apache.beam.runners.dataflow.repackaged.com.google.thirdparty</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>org.apache.beam.runners.core</pattern>
-                  <shadedPattern>org.apache.beam.runners.dataflow.repackaged.org.apache.beam.runners.core</shadedPattern>
-                </relocation>
-              </relocations>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
       <!-- Coverage analysis for unit tests. -->
       <plugin>
         <groupId>org.jacoco</groupId>
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 882657b1db4e..ea273a2d2989 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -57,6 +57,85 @@
             <testSourceDirectory>${project.basedir}/src/test/</testSourceDirectory>
           </configuration>
         </plugin>
+
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>bundle-and-repackage</id>
+              <phase>package</phase>
+              <goals>
+                <goal>shade</goal>
+              </goals>
+              <configuration>
+                <shadeTestJar>true</shadeTestJar>
+                <artifactSet>
+                  <includes>
+                    <include>com.google.guava:guava</include>
+                    <include>com.google.protobuf:protobuf-java</include>
+                    <include>net.bytebuddy:byte-buddy</include>
+                    <include>org.apache.commons:*</include>
+                  </includes>
+                </artifactSet>
+                <filters>
+                  <filter>
+                    <artifact>*:*</artifact>
+                    <excludes>
+                      <exclude>META-INF/*.SF</exclude>
+                      <exclude>META-INF/*.DSA</exclude>
+                      <exclude>META-INF/*.RSA</exclude>
+                    </excludes>
+                  </filter>
+                </filters>
+                <relocations>
+                  <relocation>
+                    <pattern>com.google.common</pattern>
+                    <excludes>
+                      <!-- com.google.common is too generic, need to exclude guava-testlib -->
+                      <exclude>com.google.common.**.testing.*</exclude>
+                    </excludes>
+                    <!--suppress MavenModelInspection -->
+                    <shadedPattern>
+                      org.apache.beam.sdk.repackaged.com.google.common
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>com.google.thirdparty</pattern>
+                    <!--suppress MavenModelInspection -->
+                    <shadedPattern>
+                      org.apache.beam.sdk.repackaged.com.google.thirdparty
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>com.google.protobuf</pattern>
+                    <!--suppress MavenModelInspection -->
+                    <shadedPattern>
+                      org.apache.beam.sdk.repackaged.com.google.protobuf
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>net.bytebuddy</pattern>
+                    <!--suppress MavenModelInspection -->
+                    <shadedPattern>
+                      org.apache.beam.sdk.repackaged.net.bytebuddy
+                    </shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.commons</pattern>
+                    <!--suppress MavenModelInspection -->
+                    <shadedPattern>
+                      org.apache.beam.sdk.repackaged.org.apache.commons
+                    </shadedPattern>
+                  </relocation>
+                </relocations>
+                <transformers>
+                  <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                </transformers>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
       </plugins>
     </pluginManagement>
 
@@ -120,92 +199,6 @@
           </execution>
         </executions>
       </plugin>
-
-      <!-- Ensure that the Maven jar plugin runs before the Maven
-        shade plugin by listing the plugin higher within the file. -->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-      </plugin>
-
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>bundle-and-repackage</id>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <shadeTestJar>true</shadeTestJar>
-              <artifactSet>
-                <includes>
-                  <include>com.google.guava:guava</include>
-                  <include>com.google.protobuf:protobuf-java</include>
-                  <include>net.bytebuddy:byte-buddy</include>
-                  <include>org.apache.commons:*</include>
-                </includes>
-              </artifactSet>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/*.SF</exclude>
-                    <exclude>META-INF/*.DSA</exclude>
-                    <exclude>META-INF/*.RSA</exclude>
-                  </excludes>
-                </filter>
-              </filters>
-              <relocations>
-                <relocation>
-                  <pattern>com.google.common</pattern>
-                  <excludes>
-                    <!-- com.google.common is too generic, need to exclude guava-testlib -->
-                    <exclude>com.google.common.**.testing.*</exclude>
-                  </excludes>
-                  <!--suppress MavenModelInspection -->
-                  <shadedPattern>
-                    org.apache.beam.sdk.repackaged.com.google.common
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.google.thirdparty</pattern>
-                  <!--suppress MavenModelInspection -->
-                  <shadedPattern>
-                    org.apache.beam.sdk.repackaged.com.google.thirdparty
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.google.protobuf</pattern>
-                  <!--suppress MavenModelInspection -->
-                  <shadedPattern>
-                    org.apache.beam.sdk.repackaged.com.google.protobuf
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>net.bytebuddy</pattern>
-                  <!--suppress MavenModelInspection -->
-                  <shadedPattern>
-                    org.apache.beam.sdk.repackaged.net.bytebuddy
-                  </shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>org.apache.commons</pattern>
-                  <!--suppress MavenModelInspection -->
-                  <shadedPattern>
-                    org.apache.beam.sdk.repackaged.org.apache.commons
-                  </shadedPattern>
-                </relocation>
-              </relocations>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 7594365bc11c..ea2d8f0cb290 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -32,6 +32,21 @@
   <packaging>jar</packaging>
 
   <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>bundle-and-repackage</id>
+              <phase>none</phase>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -42,24 +57,6 @@
           </systemPropertyVariables>
         </configuration>
       </plugin>
-
-      <!-- Ensure that the Maven jar plugin runs before the Maven
-        shade plugin by listing the plugin higher within the file. -->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-      </plugin>
-
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>bundle-and-repackage</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 

From 3efb34b8322ac5933d9d71255d67e99d8e8361d7 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Thu, 18 May 2017 19:00:00 -0700
Subject: [PATCH 261/387] update dataflow python container

---
 .../python/apache_beam/runners/dataflow/internal/dependency.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index 892d9f94d1ee..e69c8d7dac50 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -72,7 +72,8 @@
 
 # Update this version to the next version whenever there is a change that will
 # require changes to the execution environment.
-BEAM_CONTAINER_VERSION = '2.0.0'
+# This should be in the beam-[version]-[date] format, date is optional.
+BEAM_CONTAINER_VERSION = 'beam-2.1.0-20170518'
 
 # Standard file names used for staging files.
 WORKFLOW_TARBALL_FILE = 'workflow.tar.gz'

From bac7bde5f878e6b2104c2e7cf3d01894a8f1b71d Mon Sep 17 00:00:00 2001
From: Mark Liu <markliu@google.com>
Date: Wed, 17 May 2017 14:24:25 -0700
Subject: [PATCH 262/387] [BEAM-2164] Fix generate-sources.sh if directory
 contains space

---
 sdks/java/maven-archetypes/examples-java8/generate-sources.sh | 2 +-
 sdks/java/maven-archetypes/examples/generate-sources.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdks/java/maven-archetypes/examples-java8/generate-sources.sh b/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
index 58c5f22298b7..d8117ce5f266 100755
--- a/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
+++ b/sdks/java/maven-archetypes/examples-java8/generate-sources.sh
@@ -22,7 +22,7 @@
 # Usage: Invoke with no arguments from any working directory.
 
 # The directory of this script. Assumes root of the maven-archetypes module.
-HERE="$(dirname $0)"
+HERE="$( dirname "$0" )"
 
 # The directory of the examples-java and examples-java8 modules
 EXAMPLES_ROOT="${HERE}/../../../../examples/java"
diff --git a/sdks/java/maven-archetypes/examples/generate-sources.sh b/sdks/java/maven-archetypes/examples/generate-sources.sh
index d9109ac313c2..62af772884f5 100755
--- a/sdks/java/maven-archetypes/examples/generate-sources.sh
+++ b/sdks/java/maven-archetypes/examples/generate-sources.sh
@@ -21,7 +21,7 @@
 # Usage: Invoke with no arguments from any working directory.
 
 # The directory of this script. Assumes root of the maven-archetypes module.
-HERE="$(dirname $0)"
+HERE="$( dirname "$0" )"
 
 # The directory of the examples-java module
 EXAMPLES_ROOT="${HERE}/../../../../examples/java"

From c7bd2b43ddf726248291a653f8d7d796a51800a0 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 19 May 2017 11:40:20 -0700
Subject: [PATCH 263/387] Move snappy-java version to dependencyManagement

---
 pom.xml                | 7 +++++++
 sdks/java/core/pom.xml | 1 -
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index a978f58fe3b1..ed1a076b8356 100644
--- a/pom.xml
+++ b/pom.xml
@@ -141,6 +141,7 @@
     <woodstox.version>4.4.1</woodstox.version>
     <spring.version>4.3.5.RELEASE</spring.version>
     <groovy-maven-plugin.version>2.0</groovy-maven-plugin.version>
+    <snappy-java.version>1.1.4-M3</snappy-java.version>
     <surefire-plugin.version>2.20</surefire-plugin.version>
     <failsafe-plugin.version>2.20</failsafe-plugin.version>
     <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
@@ -993,6 +994,12 @@
         <version>${spring.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.xerial.snappy</groupId>
+        <artifactId>snappy-java</artifactId>
+        <version>${snappy-java.version}</version>
+      </dependency>
+
       <!-- Testing -->
 
       <dependency>
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 882657b1db4e..c5a81c012558 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -266,7 +266,6 @@
     <dependency>
       <groupId>org.xerial.snappy</groupId>
       <artifactId>snappy-java</artifactId>
-      <version>1.1.4-M3</version>
     </dependency>
 
     <dependency>

From c4d14f8be7bb72fd653f1ab8e8080fc2b65f6672 Mon Sep 17 00:00:00 2001
From: Colin Phipps <cph@moria.org.uk>
Date: Tue, 25 Apr 2017 15:28:28 +0000
Subject: [PATCH 264/387] Comply with byte limit for Datastore Commit.

---
 .../sdk/io/gcp/datastore/DatastoreV1.java     | 16 +++++++++
 .../sdk/io/gcp/datastore/DatastoreV1Test.java | 34 +++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index 16bb1b4d36a1..4cfb80119559 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -207,6 +207,13 @@ public class DatastoreV1 {
   @VisibleForTesting
   static final int DATASTORE_BATCH_UPDATE_LIMIT = 500;
 
+  /**
+   * Cloud Datastore has a limit of 10MB per RPC, so we also flush if the total size of mutations
+   * exceeds this limit. This is set lower than the 10MB limit on the RPC, as this only accounts for
+   * the mutations themselves and not the CommitRequest wrapper around them.
+   */
+  static final int DATASTORE_BATCH_UPDATE_BYTES_LIMIT = 5_000_000;
+
   /**
    * Returns an empty {@link DatastoreV1.Read} builder. Configure the source {@code projectId},
    * {@code query}, and optionally {@code namespace} and {@code numQuerySplits} using
@@ -1123,6 +1130,7 @@ static class DatastoreWriterFn extends DoFn<Mutation, Void> {
     private final V1DatastoreFactory datastoreFactory;
     // Current batch of mutations to be written.
     private final List<Mutation> mutations = new ArrayList<>();
+    private int mutationsSize = 0;  // Accumulated size of protos in mutations.
 
     private static final int MAX_RETRIES = 5;
     private static final FluentBackoff BUNDLE_WRITE_BACKOFF =
@@ -1152,7 +1160,14 @@ public void startBundle(StartBundleContext c) {
 
     @ProcessElement
     public void processElement(ProcessContext c) throws Exception {
+      Mutation write = c.element();
+      int size = write.getSerializedSize();
+      if (mutations.size() > 0
+          && mutationsSize + size >= DatastoreV1.DATASTORE_BATCH_UPDATE_BYTES_LIMIT) {
+        flushBatch();
+      }
       mutations.add(c.element());
+      mutationsSize += size;
       if (mutations.size() >= DatastoreV1.DATASTORE_BATCH_UPDATE_LIMIT) {
         flushBatch();
       }
@@ -1203,6 +1218,7 @@ private void flushBatch() throws DatastoreException, IOException, InterruptedExc
       }
       LOG.debug("Successfully wrote {} mutations", mutations.size());
       mutations.clear();
+      mutationsSize = 0;
     }
 
     @Override
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
index ba8ac846ffc7..3597b54fa16a 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
@@ -60,6 +60,7 @@
 import com.google.datastore.v1.QueryResultBatch;
 import com.google.datastore.v1.RunQueryRequest;
 import com.google.datastore.v1.RunQueryResponse;
+import com.google.datastore.v1.Value;
 import com.google.datastore.v1.client.Datastore;
 import com.google.datastore.v1.client.DatastoreException;
 import com.google.datastore.v1.client.QuerySplitter;
@@ -644,6 +645,39 @@ private void datastoreWriterFnTest(int numMutations) throws Exception {
     }
   }
 
+  /**
+   * Tests {@link DatastoreWriterFn} with large entities that need to be split into more batches.
+   */
+  @Test
+  public void testDatatoreWriterFnWithLargeEntities() throws Exception {
+    List<Mutation> mutations = new ArrayList<>();
+    for (int i = 0; i < 12; ++i) {
+      Entity.Builder entity = Entity.newBuilder().setKey(makeKey("key" + i, i + 1));
+      entity.putProperties("long", Value.newBuilder().setStringValue(new String(new char[1_000_000])
+            ).setExcludeFromIndexes(true).build());
+      mutations.add(makeUpsert(entity.build()).build());
+    }
+
+    DatastoreWriterFn datastoreWriter = new DatastoreWriterFn(StaticValueProvider.of(PROJECT_ID),
+        null, mockDatastoreFactory);
+    DoFnTester<Mutation, Void> doFnTester = DoFnTester.of(datastoreWriter);
+    doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
+    doFnTester.processBundle(mutations);
+
+    // This test is over-specific currently; it requires that we split the 12 entity writes into 3
+    // requests, but we only need each CommitRequest to be less than 10MB in size.
+    int start = 0;
+    while (start < mutations.size()) {
+      int end = Math.min(mutations.size(), start + 4);
+      CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
+      commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
+      commitRequest.addAllMutations(mutations.subList(start, end));
+      // Verify all the batch requests were made with the expected mutations.
+      verify(mockDatastore).commit(commitRequest.build());
+      start = end;
+    }
+  }
+
   /**
    * Tests {@link DatastoreV1.Read#getEstimatedSizeBytes} to fetch and return estimated size for a
    * query.

From de95c7f5bf3b8391486b898b0ffb1a0a05338725 Mon Sep 17 00:00:00 2001
From: Colin Phipps <fipsy@google.com>
Date: Tue, 9 May 2017 15:07:51 +0000
Subject: [PATCH 265/387] Make the large entities test neater.

---
 .../org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java | 1 +
 .../apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index 4cfb80119559..b198a6f568b9 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -212,6 +212,7 @@ public class DatastoreV1 {
    * exceeds this limit. This is set lower than the 10MB limit on the RPC, as this only accounts for
    * the mutations themselves and not the CommitRequest wrapper around them.
    */
+  @VisibleForTesting
   static final int DATASTORE_BATCH_UPDATE_BYTES_LIMIT = 5_000_000;
 
   /**
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
index 3597b54fa16a..460049e8b355 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
@@ -26,6 +26,7 @@
 import static com.google.datastore.v1.client.DatastoreHelper.makeOrder;
 import static com.google.datastore.v1.client.DatastoreHelper.makeUpsert;
 import static com.google.datastore.v1.client.DatastoreHelper.makeValue;
+import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.DATASTORE_BATCH_UPDATE_BYTES_LIMIT;
 import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.DATASTORE_BATCH_UPDATE_LIMIT;
 import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.DEFAULT_BUNDLE_SIZE_BYTES;
 import static org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.QUERY_BATCH_LIMIT;
@@ -60,7 +61,6 @@
 import com.google.datastore.v1.QueryResultBatch;
 import com.google.datastore.v1.RunQueryRequest;
 import com.google.datastore.v1.RunQueryResponse;
-import com.google.datastore.v1.Value;
 import com.google.datastore.v1.client.Datastore;
 import com.google.datastore.v1.client.DatastoreException;
 import com.google.datastore.v1.client.QuerySplitter;
@@ -651,9 +651,10 @@ private void datastoreWriterFnTest(int numMutations) throws Exception {
   @Test
   public void testDatatoreWriterFnWithLargeEntities() throws Exception {
     List<Mutation> mutations = new ArrayList<>();
+    int propertySize = 900_000;
     for (int i = 0; i < 12; ++i) {
       Entity.Builder entity = Entity.newBuilder().setKey(makeKey("key" + i, i + 1));
-      entity.putProperties("long", Value.newBuilder().setStringValue(new String(new char[1_000_000])
+      entity.putProperties("long", makeValue(new String(new char[propertySize])
             ).setExcludeFromIndexes(true).build());
       mutations.add(makeUpsert(entity.build()).build());
     }
@@ -666,9 +667,10 @@ public void testDatatoreWriterFnWithLargeEntities() throws Exception {
 
     // This test is over-specific currently; it requires that we split the 12 entity writes into 3
     // requests, but we only need each CommitRequest to be less than 10MB in size.
+    int propertiesPerRpc = DATASTORE_BATCH_UPDATE_BYTES_LIMIT / propertySize;
     int start = 0;
     while (start < mutations.size()) {
-      int end = Math.min(mutations.size(), start + 4);
+      int end = Math.min(mutations.size(), start + propertiesPerRpc);
       CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
       commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
       commitRequest.addAllMutations(mutations.subList(start, end));

From dcf405644e2b51303e1d2c12592fe82ee01eb32f Mon Sep 17 00:00:00 2001
From: Colin Phipps <fipsy@google.com>
Date: Tue, 9 May 2017 09:40:50 +0000
Subject: [PATCH 266/387] End-to-end test for large entity writes.

---
 .../beam/sdk/io/gcp/datastore/V1ReadIT.java   |  2 +-
 .../beam/sdk/io/gcp/datastore/V1TestUtil.java | 15 ++++++--
 .../beam/sdk/io/gcp/datastore/V1WriteIT.java  | 36 +++++++++++++++++--
 3 files changed, 47 insertions(+), 6 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
index ec7fa8f046d9..22945f599ce5 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
@@ -148,7 +148,7 @@ private static void writeEntitiesToDatastore(V1TestOptions options, String proje
     Key ancestorKey = makeAncestorKey(options.getNamespace(), options.getKind(), ancestor);
 
     for (long i = 0; i < numEntities; i++) {
-      Entity entity = makeEntity(i, ancestorKey, options.getKind(), options.getNamespace());
+      Entity entity = makeEntity(i, ancestorKey, options.getKind(), options.getNamespace(), 0);
       writer.write(entity);
     }
     writer.close();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
index dc91638878dd..5e618dfaabd2 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
@@ -92,8 +92,10 @@ static Query makeAncestorKindQuery(String kind, @Nullable String namespace, Stri
 
   /**
    * Build an entity for the given ancestorKey, kind, namespace and value.
+   * @param largePropertySize if greater than 0, add an unindexed property of the given size.
    */
-  static Entity makeEntity(Long value, Key ancestorKey, String kind, @Nullable String namespace) {
+  static Entity makeEntity(Long value, Key ancestorKey, String kind, @Nullable String namespace,
+      int largePropertySize) {
     Entity.Builder entityBuilder = Entity.newBuilder();
     Key.Builder keyBuilder = makeKey(ancestorKey, kind, UUID.randomUUID().toString());
     // NOTE: Namespace is not inherited between keys created with DatastoreHelper.makeKey, so
@@ -105,6 +107,10 @@ static Entity makeEntity(Long value, Key ancestorKey, String kind, @Nullable Str
 
     entityBuilder.setKey(keyBuilder.build());
     entityBuilder.putProperties("value", makeValue(value).build());
+    if (largePropertySize > 0) {
+      entityBuilder.putProperties("unindexed_value", makeValue(new String(
+          new char[largePropertySize]).replace("\0", "A")).setExcludeFromIndexes(true).build());
+    }
     return entityBuilder.build();
   }
 
@@ -115,18 +121,21 @@ static class CreateEntityFn extends DoFn<Long, Entity> {
     private final String kind;
     @Nullable
     private final String namespace;
+    private final int largePropertySize;
     private Key ancestorKey;
 
-    CreateEntityFn(String kind, @Nullable String namespace, String ancestor) {
+    CreateEntityFn(String kind, @Nullable String namespace, String ancestor,
+        int largePropertySize) {
       this.kind = kind;
       this.namespace = namespace;
+      this.largePropertySize = largePropertySize;
       // Build the ancestor key for all created entities once, including the namespace.
       ancestorKey = makeAncestorKey(namespace, kind, ancestor);
     }
 
     @ProcessElement
     public void processElement(ProcessContext c) throws Exception {
-      c.output(makeEntity(c.element(), ancestorKey, kind, namespace));
+      c.output(makeEntity(c.element(), ancestorKey, kind, namespace, largePropertySize));
     }
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
index 82e4d6430ab9..4a874fd7ef4d 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
@@ -67,8 +67,7 @@ public void testE2EV1Write() throws Exception {
 
     // Write to datastore
     p.apply(GenerateSequence.from(0).to(numEntities))
-        .apply(ParDo.of(new CreateEntityFn(
-            options.getKind(), options.getNamespace(), ancestor)))
+        .apply(ParDo.of(new CreateEntityFn(options.getKind(), options.getNamespace(), ancestor, 0)))
         .apply(DatastoreIO.v1().write().withProjectId(project));
 
     p.run();
@@ -79,6 +78,39 @@ public void testE2EV1Write() throws Exception {
     assertEquals(numEntitiesWritten, numEntities);
   }
 
+  /**
+   * An end-to-end test for {@link DatastoreV1.Write}.
+   *
+   * <p>Write some large test entities to Cloud Datastore, to test that a batch is flushed when
+   * the byte size limit is reached. Read and count all the entities. Verify that the count matches
+   * the number of entities written.
+   */
+  @Test
+  public void testE2EV1WriteWithLargeEntities() throws Exception {
+    Pipeline p = Pipeline.create(options);
+
+    /*
+     * Datastore has a limit of 1MB per entity, and 10MB per write RPC. If each entity is around
+     * 1MB in size, then we hit the limit on the size of the write long before we hit the limit on
+     * the number of entities per writes.
+     */
+    final int rawPropertySize = 900_000;
+    final int numLargeEntities = 100;
+
+    // Write to datastore
+    p.apply(GenerateSequence.from(0).to(numLargeEntities))
+        .apply(ParDo.of(new CreateEntityFn(
+                options.getKind(), options.getNamespace(), ancestor, rawPropertySize)))
+        .apply(DatastoreIO.v1().write().withProjectId(project));
+
+    p.run();
+
+    // Count number of entities written to datastore.
+    long numEntitiesWritten = countEntities(options, project, ancestor);
+
+    assertEquals(numEntitiesWritten, numLargeEntities);
+  }
+
   @After
   public void tearDown() throws Exception {
     deleteAllEntities(options, project, ancestor);

From 383329d6c35e50c8d907ac97522a64236bb98dd0 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Fri, 12 May 2017 16:44:10 -0700
Subject: [PATCH 267/387] [BEAM-1345] Remove
 FileSystems.setDefaultConfigInWorkers since Dataflow no longer depends on
 this

---
 runners/google-cloud-dataflow-java/pom.xml                  | 2 +-
 .../src/main/java/org/apache/beam/sdk/io/FileSystems.java   | 6 ------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index c09b7f407a5b..895a8e6a513b 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   <packaging>jar</packaging>
 
   <properties>
-    <dataflow.container_version>beam-master-20170512</dataflow.container_version>
+    <dataflow.container_version>beam-master-20170519</dataflow.container_version>
     <dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
     <dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
   </properties>
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index 1aacc9020705..2ed29e3a6718 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -449,12 +449,6 @@ static FileSystem getFileSystemInternal(String scheme) {
 
   /********************************** METHODS FOR REGISTRATION **********************************/
 
-  /** @deprecated to be removed. */
-  @Deprecated // for DataflowRunner backwards compatibility.
-  public static void setDefaultConfigInWorkers(PipelineOptions options) {
-    setDefaultPipelineOptions(options);
-  }
-
   /**
    * Sets the default configuration in workers.
    *

From 6c6d7877620af5265ed5ca14dd2ed7cd905b7404 Mon Sep 17 00:00:00 2001
From: "wtanaka.com" <wtanaka@yahoo.com>
Date: Thu, 18 May 2017 20:42:46 -1000
Subject: [PATCH 268/387] [BEAM-2293] Rename
 Top.{Largest,Smallest=>Natural,Reversed}

 - [x] Make sure the PR title is formatted like:
   `[BEAM-<Jira issue #>] Description of pull request`
 - [x] Make sure tests pass via `mvn clean verify`. (Even better, enable
       Travis-CI on your fork and ensure the whole test matrix passes).
 - [x] Replace `<Jira issue #>` in the title with the actual Jira issue
       number, if there is one.
 - [x] If this contribution is large, please file an Apache
       [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).

---
---
 .../sdk/transforms/ApproximateQuantiles.java  |  4 +--
 .../org/apache/beam/sdk/transforms/Max.java   |  4 +--
 .../org/apache/beam/sdk/transforms/Min.java   |  4 +--
 .../org/apache/beam/sdk/transforms/Top.java   | 36 +++++++++++++++----
 .../transforms/ApproximateQuantilesTest.java  |  2 +-
 .../apache/beam/sdk/transforms/MaxTest.java   |  2 +-
 .../apache/beam/sdk/transforms/MinTest.java   |  2 +-
 .../apache/beam/sdk/transforms/TopTest.java   | 10 +++---
 8 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
index d12d193edc00..e952af2bff7f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java
@@ -283,8 +283,8 @@ ApproximateQuantilesCombineFn<T, ComparatorT> create(
      * Like {@link #create(int, Comparator)}, but sorts values using their natural ordering.
      */
     public static <T extends Comparable<T>>
-        ApproximateQuantilesCombineFn<T, Top.Largest<T>> create(int numQuantiles) {
-      return create(numQuantiles, new Top.Largest<T>());
+        ApproximateQuantilesCombineFn<T, Top.Natural<T>> create(int numQuantiles) {
+      return create(numQuantiles, new Top.Natural<T>());
     }
 
     /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java
index 91851bcbe9f6..710fe77a0d70 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Max.java
@@ -158,11 +158,11 @@ BinaryCombineFn<T> of(final ComparatorT comparator) {
   }
 
   public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder(T identity) {
-    return new MaxFn<T>(identity, new Top.Largest<T>());
+    return new MaxFn<T>(identity, new Top.Natural<T>());
   }
 
   public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder() {
-    return new MaxFn<T>(null, new Top.Largest<T>());
+    return new MaxFn<T>(null, new Top.Natural<T>());
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java
index 109f4e5cb5bc..c566fb3e6ff2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Min.java
@@ -158,11 +158,11 @@ BinaryCombineFn<T> of(ComparatorT comparator) {
   }
 
   public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder(T identity) {
-    return new MinFn<T>(identity, new Top.Largest<T>());
+    return new MinFn<T>(identity, new Top.Natural<T>());
   }
 
   public static <T extends Comparable<? super T>> BinaryCombineFn<T> naturalOrder() {
-    return new MinFn<T>(null, new Top.Largest<T>());
+    return new MinFn<T>(null, new Top.Natural<T>());
   }
 
   /**
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
index 99ec49bb8ef8..319f7798e6d1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java
@@ -144,7 +144,7 @@ Combine.Globally<T, List<T>> of(int count, ComparatorT compareFn) {
    * {@code KV}s and return the top values associated with each key.
    */
   public static <T extends Comparable<T>> Combine.Globally<T, List<T>> smallest(int count) {
-    return Combine.globally(new TopCombineFn<>(count, new Smallest<T>()));
+    return Combine.globally(new TopCombineFn<>(count, new Reversed<T>()));
   }
 
   /**
@@ -188,7 +188,7 @@ public static <T extends Comparable<T>> Combine.Globally<T, List<T>> smallest(in
    * {@code KV}s and return the top values associated with each key.
    */
   public static <T extends Comparable<T>> Combine.Globally<T, List<T>> largest(int count) {
-    return Combine.globally(new TopCombineFn<>(count, new Largest<T>()));
+    return Combine.globally(new TopCombineFn<>(count, new Natural<T>()));
   }
 
   /**
@@ -281,7 +281,7 @@ public static <T extends Comparable<T>> Combine.Globally<T, List<T>> largest(int
   public static <K, V extends Comparable<V>>
       PTransform<PCollection<KV<K, V>>, PCollection<KV<K, List<V>>>>
       smallestPerKey(int count) {
-    return Combine.perKey(new TopCombineFn<>(count, new Smallest<V>()));
+    return Combine.perKey(new TopCombineFn<>(count, new Reversed<V>()));
   }
 
   /**
@@ -327,14 +327,26 @@ public static <T extends Comparable<T>> Combine.Globally<T, List<T>> largest(int
   public static <K, V extends Comparable<V>>
       PerKey<K, V, List<V>>
       largestPerKey(int count) {
-    return Combine.perKey(new TopCombineFn<>(count, new Largest<V>()));
+    return Combine.perKey(new TopCombineFn<>(count, new Natural<V>()));
+  }
+
+  /**
+   * @deprecated use {@link Natural} instead
+   */
+  @Deprecated
+  public static class Largest<T extends Comparable<? super T>>
+      implements Comparator<T>, Serializable {
+    @Override
+    public int compare(T a, T b) {
+      return a.compareTo(b);
+    }
   }
 
   /**
    * A {@code Serializable} {@code Comparator} that that uses the compared elements' natural
    * ordering.
    */
-  public static class Largest<T extends Comparable<? super T>>
+  public static class Natural<T extends Comparable<? super T>>
       implements Comparator<T>, Serializable {
     @Override
     public int compare(T a, T b) {
@@ -342,11 +354,23 @@ public int compare(T a, T b) {
     }
   }
 
+  /**
+   * @deprecated use {@link Reversed} instead
+   */
+  @Deprecated
+  public static class Smallest<T extends Comparable<? super T>>
+      implements Comparator<T>, Serializable {
+    @Override
+    public int compare(T a, T b) {
+      return b.compareTo(a);
+    }
+  }
+
   /**
    * {@code Serializable} {@code Comparator} that that uses the reverse of the compared elements'
    * natural ordering.
    */
-  public static class Smallest<T extends Comparable<? super T>>
+  public static class Reversed<T extends Comparable<? super T>>
       implements Comparator<T>, Serializable {
     @Override
     public int compare(T a, T b) {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java
index cd7898bc95a6..9e0b3cc9069e 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java
@@ -236,7 +236,7 @@ public void testAlternateComparator() {
 
   @Test
   public void testDisplayData() {
-    Top.Largest<Integer> comparer = new Top.Largest<Integer>();
+    Top.Natural<Integer> comparer = new Top.Natural<Integer>();
     PTransform<?, ?> approxQuanitiles = ApproximateQuantiles.globally(20, comparer);
     DisplayData displayData = DisplayData.from(approxQuanitiles);
 
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java
index 2b4356094894..52043e18597c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java
@@ -69,7 +69,7 @@ public void testMaxDoubleFn() {
 
   @Test
   public void testDisplayData() {
-    Top.Largest<Integer> comparer = new Top.Largest<>();
+    Top.Natural<Integer> comparer = new Top.Natural<>();
 
     Combine.Globally<Integer, Integer> max = Max.globally(comparer);
     assertThat(DisplayData.from(max), hasDisplayItem("comparer", comparer.getClass()));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java
index e89b2235b221..1ece09b532f7 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java
@@ -69,7 +69,7 @@ public void testMinDoubleFn() {
 
   @Test
   public void testDisplayData() {
-    Top.Smallest<Integer> comparer = new Top.Smallest<>();
+    Top.Reversed<Integer> comparer = new Top.Reversed<>();
 
     Combine.Globally<Integer, Integer> min = Min.globally(comparer);
     assertThat(DisplayData.from(min), hasDisplayItem("comparer", comparer.getClass()));
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java
index 9b0b27d0adbc..a7f439c2c2ee 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/TopTest.java
@@ -231,17 +231,17 @@ public void testCountConstraint() {
   @Test
   public void testTopGetNames() {
     assertEquals("Combine.globally(Top(OrderByLength))", Top.of(1, new OrderByLength()).getName());
-    assertEquals("Combine.globally(Top(Smallest))", Top.smallest(1).getName());
-    assertEquals("Combine.globally(Top(Largest))", Top.largest(2).getName());
+    assertEquals("Combine.globally(Top(Reversed))", Top.smallest(1).getName());
+    assertEquals("Combine.globally(Top(Natural))", Top.largest(2).getName());
     assertEquals("Combine.perKey(Top(IntegerComparator))",
         Top.perKey(1, new IntegerComparator()).getName());
-    assertEquals("Combine.perKey(Top(Smallest))", Top.<String, Integer>smallestPerKey(1).getName());
-    assertEquals("Combine.perKey(Top(Largest))", Top.<String, Integer>largestPerKey(2).getName());
+    assertEquals("Combine.perKey(Top(Reversed))", Top.<String, Integer>smallestPerKey(1).getName());
+    assertEquals("Combine.perKey(Top(Natural))", Top.<String, Integer>largestPerKey(2).getName());
   }
 
   @Test
   public void testDisplayData() {
-    Top.Largest<Integer> comparer = new Top.Largest<Integer>();
+    Top.Natural<Integer> comparer = new Top.Natural<Integer>();
     Combine.Globally<Integer, List<Integer>> top = Top.of(1234, comparer);
     DisplayData displayData = DisplayData.from(top);
 

From 2ed68f7f49a6f84921b0de03a57b0bbe61d263a7 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Mon, 15 May 2017 15:43:00 -0700
Subject: [PATCH 269/387] [BEAM-2253] Use resource transformer plugin to use
 versions defined in root pom.xml within maven archetype poms

---
 pom.xml                                       | 12 +++--
 .../resources/archetype-resources/pom.xml     | 51 ++++++++++++-------
 .../resources/archetype-resources/pom.xml     | 48 +++++++++++------
 .../resources/archetype-resources/pom.xml     | 12 +++--
 .../projects/basic/reference/pom.xml          | 12 +++--
 5 files changed, 90 insertions(+), 45 deletions(-)

diff --git a/pom.xml b/pom.xml
index a978f58fe3b1..211a8221290d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,8 +143,12 @@
     <groovy-maven-plugin.version>2.0</groovy-maven-plugin.version>
     <surefire-plugin.version>2.20</surefire-plugin.version>
     <failsafe-plugin.version>2.20</failsafe-plugin.version>
+    <maven-compiler-plugin.version>3.6.1</maven-compiler-plugin.version>
+    <maven-exec-plugin.version>1.4.0</maven-exec-plugin.version>
+    <maven-jar-plugin.version>3.0.2</maven-jar-plugin.version>
     <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
-    
+    <maven-shade-plugin.version>3.0.0</maven-shade-plugin.version>
+
     <compiler.error.flag>-Werror</compiler.error.flag>
     <compiler.default.pkginfo.flag>-Xpkginfo:always</compiler.default.pkginfo.flag>
     <compiler.default.exclude>nothing</compiler.default.exclude>
@@ -1131,7 +1135,7 @@
 
         <plugin>
           <artifactId>maven-compiler-plugin</artifactId>
-          <version>3.6.1</version>
+          <version>${maven-compiler-plugin.version}</version>
           <configuration>
             <source>1.7</source>
             <target>1.7</target>
@@ -1186,7 +1190,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-jar-plugin</artifactId>
-          <version>3.0.2</version>
+          <version>${maven-jar-plugin.version}</version>
           <executions>
             <execution>
               <id>default-jar</id>
@@ -1418,7 +1422,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-shade-plugin</artifactId>
-          <version>3.0.0</version>
+          <version>${maven-shade-plugin.version}</version>
           <executions>
             <execution>
               <id>bundle-and-repackage</id>
diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 5f34689f028f..47dfc91face3 100644
--- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -28,7 +28,23 @@
 
   <properties>
     <beam.version>@project.version@</beam.version>
-    <surefire-plugin.version>2.20</surefire-plugin.version>
+
+    <bigquery.version>@bigquery.version@</bigquery.version>
+    <google-clients.version>@google-clients.version@</google-clients.version>
+    <guava.version>@guava.version@</guava.version>
+    <hamcrest.version>@hamcrest.version@</hamcrest.version>
+    <jackson.version>@jackson.version@</jackson.version>
+    <joda.version>@joda.version@</joda.version>
+    <junit.version>@junit.version@</junit.version>
+    <maven-compiler-plugin.version>@maven-compiler-plugin.version@</maven-compiler-plugin.version>
+    <maven-exec-plugin.version>@maven-exec-plugin.version@</maven-exec-plugin.version>
+    <maven-jar-plugin.version>@maven-jar-plugin.version@</maven-jar-plugin.version>
+    <maven-shade-plugin.version>@maven-shade-plugin.version@</maven-shade-plugin.version>
+    <mockito.version>@mockito.version@</mockito.version>
+    <pubsub.version>@pubsub.version@</pubsub.version>
+    <slf4j.version>@slf4j.version@</slf4j.version>
+    <spark.version>@spark.version@</spark.version>
+    <surefire-plugin.version>@surefire-plugin.version@</surefire-plugin.version>
   </properties>
 
   <repositories>
@@ -50,7 +66,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.5.1</version>
+        <version>${maven-compiler-plugin.version}</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>
@@ -80,6 +96,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
+        <version>${maven-jar-plugin.version}</version>
       </plugin>
 
       <!--
@@ -89,7 +106,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven-shade-plugin.version}</version>
         <executions>
           <execution>
             <phase>package</phase>
@@ -122,7 +139,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.4.0</version>
+          <version>${maven-exec-plugin.version}</version>
           <configuration>
             <cleanupDaemonThreads>false</cleanupDaemonThreads>
           </configuration>
@@ -224,7 +241,7 @@
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-streaming_2.10</artifactId>
-          <version>1.6.2</version>
+          <version>${spark.version}</version>
           <scope>runtime</scope>
           <exclusions>
             <exclusion>
@@ -236,7 +253,7 @@
         <dependency>
           <groupId>com.fasterxml.jackson.module</groupId>
           <artifactId>jackson-module-scala_2.10</artifactId>
-          <version>@jackson.version@</version>
+          <version>${jackson.version}</version>
           <scope>runtime</scope>
         </dependency>
       </dependencies>
@@ -261,7 +278,7 @@
     <dependency>
       <groupId>com.google.api-client</groupId>
       <artifactId>google-api-client</artifactId>
-      <version>1.22.0</version>
+      <version>${google-clients.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -276,7 +293,7 @@
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-bigquery</artifactId>
-      <version>v2-rev295-1.22.0</version>
+      <version>${bigquery.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -290,7 +307,7 @@
     <dependency>
       <groupId>com.google.http-client</groupId>
       <artifactId>google-http-client</artifactId>
-      <version>1.22.0</version>
+      <version>${google-clients.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -304,7 +321,7 @@
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-pubsub</artifactId>
-      <version>v1-rev10-1.22.0</version>
+      <version>${pubsub.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -318,26 +335,26 @@
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
-      <version>2.4</version>
+      <version>${joda.version}</version>
     </dependency>
 
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>20.0</version>
+      <version>${guava.version}</version>
     </dependency>
 
     <!-- Add slf4j API frontend binding with JUL backend -->
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
     </dependency>
 
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-jdk14</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
       <!-- When loaded at runtime this will wire up slf4j to the JUL backend -->
       <scope>runtime</scope>
     </dependency>
@@ -347,19 +364,19 @@
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest-all</artifactId>
-      <version>1.3</version>
+      <version>${hamcrest.version}</version>
     </dependency>
 
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <version>4.12</version>
+      <version>${junit.version}</version>
     </dependency>
 
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-all</artifactId>
-      <version>1.9.5</version>
+      <version>${mockito.version}</version>
       <scope>test</scope>
     </dependency>
 
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index a3d7b8fd73b6..025e3565a21c 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -28,7 +28,22 @@
 
   <properties>
     <beam.version>@project.version@</beam.version>
-    <surefire-plugin.version>2.20</surefire-plugin.version>
+
+    <bigquery.version>@bigquery.version@</bigquery.version>
+    <google-clients.version>@google-clients.version@</google-clients.version>
+    <guava.version>@guava.version@</guava.version>
+    <hamcrest.version>@hamcrest.version@</hamcrest.version>
+    <jackson.version>@jackson.version@</jackson.version>
+    <joda.version>@joda.version@</joda.version>
+    <junit.version>@junit.version@</junit.version>
+    <maven-compiler-plugin.version>@maven-compiler-plugin.version@</maven-compiler-plugin.version>
+    <maven-exec-plugin.version>@maven-exec-plugin.version@</maven-exec-plugin.version>
+    <maven-jar-plugin.version>@maven-jar-plugin.version@</maven-jar-plugin.version>
+    <maven-shade-plugin.version>@maven-shade-plugin.version@</maven-shade-plugin.version>
+    <pubsub.version>@pubsub.version@</pubsub.version>
+    <slf4j.version>@slf4j.version@</slf4j.version>
+    <spark.version>@spark.version@</spark.version>
+    <surefire-plugin.version>@surefire-plugin.version@</surefire-plugin.version>
   </properties>
 
   <repositories>
@@ -50,7 +65,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.5.1</version>
+        <version>${maven-compiler-plugin.version}</version>
         <configuration>
           <source>${targetPlatform}</source>
           <target>${targetPlatform}</target>
@@ -80,6 +95,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
+        <version>${maven-jar-plugin.version}</version>
       </plugin>
 
       <!--
@@ -89,7 +105,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven-shade-plugin.version}</version>
         <executions>
           <execution>
             <phase>package</phase>
@@ -122,7 +138,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.4.0</version>
+          <version>${maven-exec-plugin.version}</version>
           <configuration>
             <cleanupDaemonThreads>false</cleanupDaemonThreads>
           </configuration>
@@ -224,7 +240,7 @@
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-streaming_2.10</artifactId>
-          <version>1.6.2</version>
+          <version>${spark.version}</version>
           <scope>runtime</scope>
           <exclusions>
             <exclusion>
@@ -236,7 +252,7 @@
         <dependency>
           <groupId>com.fasterxml.jackson.module</groupId>
           <artifactId>jackson-module-scala_2.10</artifactId>
-          <version>@jackson.version@</version>
+          <version>${jackson.version}</version>
           <scope>runtime</scope>
         </dependency>
       </dependencies>
@@ -262,7 +278,7 @@
     <dependency>
       <groupId>com.google.api-client</groupId>
       <artifactId>google-api-client</artifactId>
-      <version>1.22.0</version>
+      <version>${google-clients.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -276,7 +292,7 @@
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-bigquery</artifactId>
-      <version>v2-rev295-1.22.0</version>
+      <version>${bigquery.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -290,7 +306,7 @@
     <dependency>
       <groupId>com.google.http-client</groupId>
       <artifactId>google-http-client</artifactId>
-      <version>1.22.0</version>
+      <version>${google-clients.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -304,7 +320,7 @@
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-pubsub</artifactId>
-      <version>v1-rev10-1.22.0</version>
+      <version>${pubsub.version}</version>
       <exclusions>
         <!-- Exclude an old version of guava that is being pulled
              in by a transitive dependency of google-api-client -->
@@ -318,26 +334,26 @@
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
-      <version>2.4</version>
+      <version>${joda.version}</version>
     </dependency>
 
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>20.0</version>
+      <version>${guava.version}</version>
     </dependency>
 
     <!-- Add slf4j API frontend binding with JUL backend -->
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
     </dependency>
 
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-jdk14</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
       <!-- When loaded at runtime this will wire up slf4j to the JUL backend -->
       <scope>runtime</scope>
     </dependency>
@@ -347,13 +363,13 @@
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest-all</artifactId>
-      <version>1.3</version>
+      <version>${hamcrest.version}</version>
     </dependency>
 
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <version>4.12</version>
+      <version>${junit.version}</version>
     </dependency>
 
     <!-- The DirectRunner is needed for unit tests. -->
diff --git a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
index dddd5ab8a819..f4fb9f8df26f 100644
--- a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/pom.xml
@@ -26,6 +26,10 @@
 
   <properties>
     <beam.version>@project.version@</beam.version>
+
+    <maven-compiler-plugin.version>@maven-compiler-plugin.version@</maven-compiler-plugin.version>
+    <maven-exec-plugin.version>@maven-exec-plugin.version@</maven-exec-plugin.version>
+    <slf4j.version>@slf4j.version@</slf4j.version>
   </properties>
 
   <repositories>
@@ -47,7 +51,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.5.1</version>
+        <version>${maven-compiler-plugin.version}</version>
         <configuration>
           <source>${targetPlatform}</source>
           <target>${targetPlatform}</target>
@@ -60,7 +64,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.4.0</version>
+          <version>${maven-exec-plugin.version}</version>
           <configuration>
             <cleanupDaemonThreads>false</cleanupDaemonThreads>
           </configuration>
@@ -94,12 +98,12 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-jdk14</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
     </dependency>
   </dependencies>
 </project>
diff --git a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
index 39fefd6ff60a..60405e6dbb17 100644
--- a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
+++ b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/pom.xml
@@ -26,6 +26,10 @@
 
   <properties>
     <beam.version>@project.version@</beam.version>
+
+    <maven-compiler-plugin.version>3.6.1</maven-compiler-plugin.version>
+    <maven-exec-plugin.version>1.4.0</maven-exec-plugin.version>
+    <slf4j.version>1.7.14</slf4j.version>
   </properties>
 
   <repositories>
@@ -47,7 +51,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.5.1</version>
+        <version>${maven-compiler-plugin.version}</version>
         <configuration>
           <source>1.7</source>
           <target>1.7</target>
@@ -60,7 +64,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.4.0</version>
+          <version>${maven-exec-plugin.version}</version>
           <configuration>
             <cleanupDaemonThreads>false</cleanupDaemonThreads>
           </configuration>
@@ -94,12 +98,12 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-jdk14</artifactId>
-      <version>1.7.14</version>
+      <version>${slf4j.version}</version>
     </dependency>
   </dependencies>
 </project>

From 5c163c77bd6d948231ea5a5ce9cce7a1064b0331 Mon Sep 17 00:00:00 2001
From: Romain Yon <yonromai@users.noreply.github.com>
Date: Fri, 19 May 2017 16:16:24 -0400
Subject: [PATCH 270/387] [BEAM-2334] Fix OutOfMemoryError

---
 .../org/apache/beam/runners/dataflow/util/RandomAccessData.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
index 5ea9f07ff6a9..0c0890243d25 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
@@ -350,7 +350,7 @@ private void ensureCapacity(int minCapacity) {
 
     // Try to double the size of the buffer, if thats not enough, just use the new capacity.
     // Note that we use Math.min(long, long) to not cause overflow on the multiplication.
-    int newCapacity = (int) Math.min(Integer.MAX_VALUE, buffer.length * 2L);
+    int newCapacity = (int) Math.min(Integer.MAX_VALUE - 8, buffer.length * 2L);
     if (newCapacity < minCapacity) {
         newCapacity = minCapacity;
     }

From 4a63970261be22660f5fef8bf37b5d18301315ef Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 19 May 2017 15:24:19 -0700
Subject: [PATCH 271/387] Add TransformPayloadTranslatorRegistrar

---
 runners/core-construction-java/pom.xml        |  7 ++++-
 .../core/construction/PTransforms.java        | 21 +++++++++-----
 .../runners/core/construction/ParDos.java     | 15 ++++++++++
 .../TransformPayloadTranslatorRegistrar.java  | 29 +++++++++++++++++++
 4 files changed, 64 insertions(+), 8 deletions(-)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java

diff --git a/runners/core-construction-java/pom.xml b/runners/core-construction-java/pom.xml
index abf0b6512fec..7eaa6f35b2f9 100644
--- a/runners/core-construction-java/pom.xml
+++ b/runners/core-construction-java/pom.xml
@@ -89,6 +89,12 @@
       <artifactId>slf4j-api</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.auto.service</groupId>
+      <artifactId>auto-service</artifactId>
+      <optional>true</optional>
+    </dependency>
+
     <dependency>
       <groupId>com.google.auto.value</groupId>
       <artifactId>auto-value</artifactId>
@@ -114,6 +120,5 @@
       <artifactId>mockito-all</artifactId>
       <scope>test</scope>
     </dependency>
-
   </dependencies>
 </project>
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
index 16276b9ed82c..9826b77b300a 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
@@ -24,12 +24,11 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
-import org.apache.beam.runners.core.construction.ParDos.ParDoPayloadTranslator;
+import java.util.ServiceLoader;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
@@ -40,11 +39,19 @@
  */
 public class PTransforms {
   private static final Map<Class<? extends PTransform>, TransformPayloadTranslator>
-      KNOWN_PAYLOAD_TRANSLATORS =
-          ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder()
-              .put(ParDo.MultiOutput.class, ParDoPayloadTranslator.create())
-              .build();
-  // TODO: Load via service loader.
+      KNOWN_PAYLOAD_TRANSLATORS = loadTransformPayloadTranslators();
+
+  private static Map<Class<? extends PTransform>, TransformPayloadTranslator>
+      loadTransformPayloadTranslators() {
+    ImmutableMap.Builder<Class<? extends PTransform>, TransformPayloadTranslator> builder =
+        ImmutableMap.builder();
+    for (TransformPayloadTranslatorRegistrar registrar :
+        ServiceLoader.load(TransformPayloadTranslatorRegistrar.class)) {
+      builder.putAll(registrar.getTransformPayloadTranslators());
+    }
+    return builder.build();
+  }
+
   private PTransforms() {}
 
   /**
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
index b2b29df65dc0..4752bd1a8940 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
@@ -20,6 +20,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
 import com.google.common.base.Optional;
 import com.google.protobuf.Any;
@@ -28,6 +29,7 @@
 import com.google.protobuf.InvalidProtocolBufferException;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import org.apache.beam.runners.core.construction.PTransforms.TransformPayloadTranslator;
@@ -46,6 +48,7 @@
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Materializations;
+import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
 import org.apache.beam.sdk.transforms.ViewFn;
@@ -107,6 +110,18 @@ public FunctionSpec translate(
           .setParameter(Any.pack(payload))
           .build();
     }
+
+    /**
+     * Registers {@link ParDoPayloadTranslator}.
+     */
+    @AutoService(TransformPayloadTranslatorRegistrar.class)
+    public static class Registrar implements TransformPayloadTranslatorRegistrar {
+      @Override
+      public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+          getTransformPayloadTranslators() {
+        return Collections.singletonMap(ParDo.MultiOutput.class, new ParDoPayloadTranslator());
+      }
+    }
   }
 
   public static ParDoPayload toProto(ParDo.MultiOutput<?, ?> parDo, SdkComponents components) {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
new file mode 100644
index 000000000000..bc568a616159
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import java.util.Map;
+import org.apache.beam.runners.core.construction.PTransforms.TransformPayloadTranslator;
+import org.apache.beam.sdk.transforms.PTransform;
+
+/** A registrar of TransformPayloadTranslator. */
+public interface TransformPayloadTranslatorRegistrar {
+  Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+      getTransformPayloadTranslators();
+}

From 5e9fcebc07725de368391914781e5b4d5f9c4a19 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 19 May 2017 12:57:41 -0700
Subject: [PATCH 272/387] Remove Pipeline reference from TransformHierarchy

This change removes a direct dependency cycle between Pipeline and
TransformHierarchy. There is still an indirect cycle through PValues, but that
is slightly less problematic.
---
 .../translation/ApexPipelineTranslator.java   |  4 +-
 .../apex/translation/TranslationContext.java  |  5 +-
 .../core/construction/SdkComponents.java      | 14 +++---
 .../core/construction/SdkComponentsTest.java  |  7 +--
 .../runners/direct/DirectGraphVisitor.java    |  3 +-
 .../direct/KeyedPValueTrackingVisitor.java    |  5 +-
 .../flink/FlinkBatchPipelineTranslator.java   |  2 +-
 .../beam/runners/flink/FlinkRunner.java       | 10 +---
 .../FlinkStreamingPipelineTranslator.java     |  4 +-
 .../dataflow/DataflowPipelineTranslator.java  |  6 +--
 .../beam/runners/dataflow/DataflowRunner.java |  2 +-
 .../beam/runners/spark/SparkRunner.java       |  2 +-
 .../streaming/TrackStreamingSourcesTest.java  | 14 +++++-
 .../java/org/apache/beam/sdk/Pipeline.java    | 50 +++++++++++++++----
 .../beam/sdk/runners/TransformHierarchy.java  |  6 +--
 .../sdk/runners/TransformHierarchyTest.java   |  2 +-
 16 files changed, 82 insertions(+), 54 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
index 32e470f6029a..bda074b0a29b 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
@@ -49,7 +49,7 @@
  * into Apex logical plan {@link DAG}.
  */
 @SuppressWarnings({"rawtypes", "unchecked"})
-public class ApexPipelineTranslator implements Pipeline.PipelineVisitor {
+public class ApexPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
   private static final Logger LOG = LoggerFactory.getLogger(ApexPipelineTranslator.class);
 
   /**
@@ -110,7 +110,7 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
       throw new UnsupportedOperationException(
           "no translator registered for " + transform);
     }
-    translationContext.setCurrentTransform(node);
+    translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
     translator.translate(transform, translationContext);
   }
 
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index a5e30281a9e3..aff3863624c4 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -36,7 +36,6 @@
 import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -77,8 +76,8 @@ public <InputT extends PInput> InputT getViewInput(PCollectionView<?> view) {
     this.pipelineOptions = pipelineOptions;
   }
 
-  public void setCurrentTransform(TransformHierarchy.Node treeNode) {
-    this.currentTransform = treeNode.toAppliedPTransform();
+  public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
+    this.currentTransform = transform;
   }
 
   public ApexPipelineOptions getPipelineOptions() {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index eb29b9a3ae7a..5714fc510481 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -62,10 +62,10 @@ static SdkComponents create() {
     return new SdkComponents();
   }
 
-  public static RunnerApi.Pipeline translatePipeline(Pipeline p) {
+  public static RunnerApi.Pipeline translatePipeline(Pipeline pipeline) {
     final SdkComponents components = create();
     final Collection<String> rootIds = new HashSet<>();
-    p.traverseTopologically(
+    pipeline.traverseTopologically(
         new PipelineVisitor.Defaults() {
           private final ListMultimap<Node, AppliedPTransform<?, ?, ?>> children =
               ArrayListMultimap.create();
@@ -77,9 +77,10 @@ public void leaveCompositeTransform(Node node) {
                 rootIds.add(components.getExistingPTransformId(pipelineRoot));
               }
             } else {
-              children.put(node.getEnclosingNode(), node.toAppliedPTransform());
+              children.put(node.getEnclosingNode(), node.toAppliedPTransform(getPipeline()));
               try {
-                components.registerPTransform(node.toAppliedPTransform(), children.get(node));
+                components.registerPTransform(
+                    node.toAppliedPTransform(getPipeline()), children.get(node));
               } catch (IOException e) {
                 throw new RuntimeException(e);
               }
@@ -88,10 +89,11 @@ public void leaveCompositeTransform(Node node) {
 
           @Override
           public void visitPrimitiveTransform(Node node) {
-            children.put(node.getEnclosingNode(), node.toAppliedPTransform());
+            children.put(node.getEnclosingNode(), node.toAppliedPTransform(getPipeline()));
             try {
               components.registerPTransform(
-                  node.toAppliedPTransform(), Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+                  node.toAppliedPTransform(getPipeline()),
+                  Collections.<AppliedPTransform<?, ?, ?>>emptyList());
             } catch (IOException e) {
               throw new IllegalStateException(e);
             }
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
index 7424886d1009..55702ea22390 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SdkComponentsTest.java
@@ -97,17 +97,12 @@ public void translatePipeline() {
 
     final RunnerApi.Pipeline pipelineProto = SdkComponents.translatePipeline(pipeline);
     pipeline.traverseTopologically(
-        new PipelineVisitor() {
+        new PipelineVisitor.Defaults() {
           Set<Node> transforms = new HashSet<>();
           Set<PCollection<?>> pcollections = new HashSet<>();
           Set<Equivalence.Wrapper<? extends Coder<?>>> coders = new HashSet<>();
           Set<WindowingStrategy<?, ?>> windowingStrategies = new HashSet<>();
 
-          @Override
-          public CompositeBehavior enterCompositeTransform(Node node) {
-            return CompositeBehavior.ENTER_TRANSFORM;
-          }
-
           @Override
           public void leaveCompositeTransform(Node node) {
             if (node.isRootNode()) {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index 1ee8ceb9a7e5..01204e3049dd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -41,6 +41,7 @@
  * input after the upstream transform has produced and committed output.
  */
 class DirectGraphVisitor extends PipelineVisitor.Defaults {
+
   private Map<POutput, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
 
   private ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers =
@@ -101,7 +102,7 @@ public void visitValue(PValue value, TransformHierarchy.Node producer) {
 
   private AppliedPTransform<?, ?, ?> getAppliedTransform(TransformHierarchy.Node node) {
     @SuppressWarnings({"rawtypes", "unchecked"})
-    AppliedPTransform<?, ?, ?> application = node.toAppliedPTransform();
+    AppliedPTransform<?, ?, ?> application = node.toAppliedPTransform(getPipeline());
     return application;
   }
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
index 347f313af85a..f9b2daeaa1df 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java
@@ -44,7 +44,7 @@
  */
 // TODO: Handle Key-preserving transforms when appropriate and more aggressively make PTransforms
 // unkeyed
-class KeyedPValueTrackingVisitor implements PipelineVisitor {
+class KeyedPValueTrackingVisitor extends PipelineVisitor.Defaults {
 
   private static final Set<Class<? extends PTransform>> PRODUCES_KEYED_OUTPUTS =
       ImmutableSet.of(
@@ -90,9 +90,6 @@ public void leaveCompositeTransform(TransformHierarchy.Node node) {
     }
   }
 
-  @Override
-  public void visitPrimitiveTransform(TransformHierarchy.Node node) {}
-
   @Override
   public void visitValue(PValue value, TransformHierarchy.Node producer) {
     boolean inputsAreKeyed = true;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java
index 854b67460ae2..50910b5bab5d 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java
@@ -112,7 +112,7 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
     BatchTransformTranslator<T> typedTranslator = (BatchTransformTranslator<T>) translator;
 
     // create the applied PTransform on the batchContext
-    batchContext.setCurrentTransform(node.toAppliedPTransform());
+    batchContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
     typedTranslator.translateNode(typedTransform, batchContext);
   }
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
index 80ef7bb32acd..ca12615be03c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java
@@ -38,7 +38,6 @@
 import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.values.PValue;
 import org.apache.flink.api.common.JobExecutionResult;
 import org.apache.flink.client.program.DetachedEnvironment;
 import org.slf4j.Logger;
@@ -199,10 +198,7 @@ private void logWarningIfPCollectionViewHasNonDeterministicKeyCoder(Pipeline pip
     // have just recorded the full names during apply time.
     if (!ptransformViewsWithNonDeterministicKeyCoders.isEmpty()) {
       final SortedSet<String> ptransformViewNamesWithNonDeterministicKeyCoders = new TreeSet<>();
-      pipeline.traverseTopologically(new Pipeline.PipelineVisitor() {
-        @Override
-        public void visitValue(PValue value, TransformHierarchy.Node producer) {
-        }
+      pipeline.traverseTopologically(new Pipeline.PipelineVisitor.Defaults() {
 
         @Override
         public void visitPrimitiveTransform(TransformHierarchy.Node node) {
@@ -218,10 +214,6 @@ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
           }
           return CompositeBehavior.ENTER_TRANSFORM;
         }
-
-        @Override
-        public void leaveCompositeTransform(TransformHierarchy.Node node) {
-        }
       });
 
       LOG.warn("Unable to use indexed implementation for View.AsMap and View.AsMultimap for {} "
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
index 53a1fa160ad1..8da68c5fc11e 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
@@ -188,7 +188,7 @@ public void visitValue(PValue value, TransformHierarchy.Node producer) {
     StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 
     // create the applied PTransform on the streamingContext
-    streamingContext.setCurrentTransform(node.toAppliedPTransform());
+    streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
     typedTranslator.translateNode(typedTransform, streamingContext);
   }
 
@@ -203,7 +203,7 @@ public void visitValue(PValue value, TransformHierarchy.Node producer) {
     @SuppressWarnings("unchecked")
     StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 
-    streamingContext.setCurrentTransform(node.toAppliedPTransform());
+    streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
 
     return typedTranslator.canTranslate(typedTransform, streamingContext);
   }
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 840bda846ed6..6d7a0f847ca2 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -431,18 +431,18 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
           transform,
           node.getFullName());
       LOG.debug("Translating {}", transform);
-      currentTransform = node.toAppliedPTransform();
+      currentTransform = node.toAppliedPTransform(getPipeline());
       translator.translate(transform, this);
       currentTransform = null;
     }
 
     @Override
     public void visitValue(PValue value, TransformHierarchy.Node producer) {
-      producers.put(value, producer.toAppliedPTransform());
+      producers.put(value, producer.toAppliedPTransform(getPipeline()));
       LOG.debug("Checking translation of {}", value);
       if (!producer.isCompositeNode()) {
         // Primitive transforms are the only ones assigned step names.
-        asOutputReference(value, producer.toAppliedPTransform());
+        asOutputReference(value, producer.toAppliedPTransform(getPipeline()));
       }
     }
 
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 2ef87374956d..cce6ce79b883 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -729,7 +729,7 @@ private void logWarningIfPCollectionViewHasNonDeterministicKeyCoder(Pipeline pip
     if (!ptransformViewsWithNonDeterministicKeyCoders.isEmpty()) {
       final SortedSet<String> ptransformViewNamesWithNonDeterministicKeyCoders = new TreeSet<>();
       pipeline.traverseTopologically(
-          new PipelineVisitor() {
+          new PipelineVisitor.Defaults() {
             @Override
             public void visitValue(PValue value, TransformHierarchy.Node producer) {}
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 8c02f0f0e90d..9e2426ef8381 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -404,7 +404,7 @@ <TransformT extends PTransform<? super PInput, POutput>> void doVisitTransform(
       @SuppressWarnings("unchecked")
       TransformEvaluator<TransformT> evaluator = translate(node, transform, transformClass);
       LOG.info("Evaluating {}", transform);
-      AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform();
+      AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform(getPipeline());
       ctxt.setCurrentTransform(appliedTransform);
       evaluator.evaluate(transform, ctxt);
       ctxt.setCurrentTransform(null);
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index 33a636ad65f8..e8a59510cdb9 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -147,6 +147,12 @@ private void assertSourceIds(List<Integer> streamingSources) {
       assertThat(streamingSources, containsInAnyOrder(expected));
     }
 
+    @Override
+    public void enterPipeline(Pipeline p) {
+      super.enterPipeline(p);
+      evaluator.enterPipeline(p);
+    }
+
     @Override
     public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
       return evaluator.enterCompositeTransform(node);
@@ -156,7 +162,7 @@ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
     public void visitPrimitiveTransform(TransformHierarchy.Node node) {
       PTransform transform = node.getTransform();
       if (transform.getClass() == transformClassToAssert) {
-        AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform();
+        AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform(getPipeline());
         ctxt.setCurrentTransform(appliedTransform);
         //noinspection unchecked
         Dataset dataset = ctxt.borrowDataset((PTransform<? extends PValue, ?>) transform);
@@ -166,6 +172,12 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
         evaluator.visitPrimitiveTransform(node);
       }
     }
+
+    @Override
+    public void leavePipeline(Pipeline p) {
+      super.leavePipeline(p);
+      evaluator.leavePipeline(p);
+    }
   }
 
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index 83496a5302ad..bdf8a12aa2bf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -205,7 +206,7 @@ private void checkNoMoreMatches(final List<PTransformOverride> overrides) {
           public CompositeBehavior enterCompositeTransform(Node node) {
             if (!node.isRootNode()) {
               for (PTransformOverride override : overrides) {
-                if (override.getMatcher().matches(node.toAppliedPTransform())) {
+                if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
                   matched.put(node, override);
                 }
               }
@@ -227,7 +228,7 @@ public void leaveCompositeTransform(Node node) {
           @Override
           public void visitPrimitiveTransform(Node node) {
             for (PTransformOverride override : overrides) {
-              if (override.getMatcher().matches(node.toAppliedPTransform())) {
+              if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
                 matched.put(node, override);
               }
             }
@@ -238,7 +239,7 @@ public void visitPrimitiveTransform(Node node) {
   private void replace(final PTransformOverride override) {
     final Set<Node> matches = new HashSet<>();
     final Set<Node> freedNodes = new HashSet<>();
-    transforms.visit(
+    traverseTopologically(
         new PipelineVisitor.Defaults() {
           @Override
           public CompositeBehavior enterCompositeTransform(Node node) {
@@ -247,7 +248,8 @@ public CompositeBehavior enterCompositeTransform(Node node) {
               freedNodes.add(node);
               return CompositeBehavior.ENTER_TRANSFORM;
             }
-            if (!node.isRootNode() && override.getMatcher().matches(node.toAppliedPTransform())) {
+            if (!node.isRootNode()
+                && override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
               matches.add(node);
               // This node will be freed. When we visit any of its children, they will also be freed
               freedNodes.add(node);
@@ -259,7 +261,7 @@ public CompositeBehavior enterCompositeTransform(Node node) {
           public void visitPrimitiveTransform(Node node) {
             if (freedNodes.contains(node.getEnclosingNode())) {
               freedNodes.add(node);
-            } else if (override.getMatcher().matches(node.toAppliedPTransform())) {
+            } else if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) {
               matches.add(node);
               freedNodes.add(node);
             }
@@ -334,8 +336,14 @@ public void setCoderRegistry(CoderRegistry coderRegistry) {
   @Internal
   public interface PipelineVisitor {
     /**
-     * Called for each composite transform after all topological predecessors have been visited
-     * but before any of its component transforms.
+     * Called before visiting anything values or transforms, as many uses of a visitor require
+     * access to the {@link Pipeline} object itself.
+     */
+    void enterPipeline(Pipeline p);
+
+    /**
+     * Called for each composite transform after all topological predecessors have been visited but
+     * before any of its component transforms.
      *
      * <p>The return value controls whether or not child transforms are visited.
      */
@@ -359,6 +367,11 @@ public interface PipelineVisitor {
      */
     void visitValue(PValue value, TransformHierarchy.Node producer);
 
+    /**
+     * Called when all values and transforms in a {@link Pipeline} have been visited.
+     */
+    void leavePipeline(Pipeline pipeline);
+
     /**
      * Control enum for indicating whether or not a traversal should process the contents of
      * a composite transform or not.
@@ -373,6 +386,18 @@ enum CompositeBehavior {
      * User implementations can override just those methods they are interested in.
      */
     class Defaults implements PipelineVisitor {
+
+      private Pipeline pipeline;
+
+      protected Pipeline getPipeline() {
+        return pipeline;
+      }
+
+      @Override
+      public void enterPipeline(Pipeline pipeline) {
+        this.pipeline = checkNotNull(pipeline);
+      }
+
       @Override
       public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
         return CompositeBehavior.ENTER_TRANSFORM;
@@ -386,6 +411,11 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) { }
 
       @Override
       public void visitValue(PValue value, TransformHierarchy.Node producer) { }
+
+      @Override
+      public void leavePipeline(Pipeline pipeline) {
+        this.pipeline = null;
+      }
     }
   }
 
@@ -406,7 +436,9 @@ public void visitValue(PValue value, TransformHierarchy.Node producer) { }
    */
   @Internal
   public void traverseTopologically(PipelineVisitor visitor) {
+    visitor.enterPipeline(this);
     transforms.visit(visitor);
+    visitor.leavePipeline(this);
   }
 
   /**
@@ -444,7 +476,7 @@ OutputT applyTransform(String name, InputT input,
   /////////////////////////////////////////////////////////////////////////////
   // Below here are internal operations, never called by users.
 
-  private final TransformHierarchy transforms = new TransformHierarchy(this);
+  private final TransformHierarchy transforms = new TransformHierarchy();
   private Set<String> usedFullNames = new HashSet<>();
   private CoderRegistry coderRegistry;
   private final List<String> unstableNames = new ArrayList<>();
@@ -495,7 +527,7 @@ void applyReplacement(
           PTransformOverrideFactory<InputT, OutputT, TransformT> replacementFactory) {
     PTransformReplacement<InputT, OutputT> replacement =
         replacementFactory.getReplacementTransform(
-            (AppliedPTransform<InputT, OutputT, TransformT>) original.toAppliedPTransform());
+            (AppliedPTransform<InputT, OutputT, TransformT>) original.toAppliedPTransform(this));
     if (replacement.getTransform() == original.getTransform()) {
       return;
     }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index fac558bc8432..2f0e8efd7de8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -56,7 +56,6 @@
 public class TransformHierarchy {
   private static final Logger LOG = LoggerFactory.getLogger(TransformHierarchy.class);
 
-  private final Pipeline pipeline;
   private final Node root;
   private final Map<Node, PInput> unexpandedInputs;
   private final Map<POutput, Node> producers;
@@ -65,8 +64,7 @@ public class TransformHierarchy {
   // Maintain a stack based on the enclosing nodes
   private Node current;
 
-  public TransformHierarchy(Pipeline pipeline) {
-    this.pipeline = pipeline;
+  public TransformHierarchy() {
     producers = new HashMap<>();
     producerInput = new HashMap<>();
     unexpandedInputs = new HashMap<>();
@@ -453,7 +451,7 @@ public Map<TupleTag<?>, PValue> getOutputs() {
     /**
      * Returns the {@link AppliedPTransform} representing this {@link Node}.
      */
-    public AppliedPTransform<?, ?, ?> toAppliedPTransform() {
+    public AppliedPTransform<?, ?, ?> toAppliedPTransform(Pipeline pipeline) {
       return AppliedPTransform.of(
           getFullName(), inputs, outputs, (PTransform) getTransform(), pipeline);
     }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
index 125e15902d17..1197d1b04eb6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
@@ -79,7 +79,7 @@ public class TransformHierarchyTest implements Serializable {
 
   @Before
   public void setup() {
-    hierarchy = new TransformHierarchy(pipeline);
+    hierarchy = new TransformHierarchy();
   }
 
   @Test

From a7ad0c2b06df5fc281eb91f73053396a61b11f53 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 18 May 2017 09:09:33 -0700
Subject: [PATCH 273/387] Fix GroupAlsoByWindowEvaluatorFactory doc

---
 .../GroupAlsoByWindowEvaluatorFactory.java     | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 84be15dd19cd..78ef7fe5d6da 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -26,8 +26,6 @@
 import java.util.Collection;
 import org.apache.beam.runners.core.GroupAlsoByWindowsAggregators;
 import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly;
-import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupAlsoByWindow;
-import org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.OutputWindowedValue;
 import org.apache.beam.runners.core.ReduceFnRunner;
@@ -57,7 +55,7 @@
 
 /**
  * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
- * {@link GroupByKeyOnly} {@link PTransform}.
+ * {@link DirectGroupAlsoByWindow} {@link PTransform}.
  */
 class GroupAlsoByWindowEvaluatorFactory implements TransformEvaluatorFactory {
   private final EvaluationContext evaluationContext;
@@ -92,8 +90,9 @@ private <K, V> TransformEvaluator<KeyedWorkItem<K, V>> createEvaluator(
   }
 
   /**
-   * A transform evaluator for the pseudo-primitive {@link GroupAlsoByWindow}. Windowing is ignored;
-   * all input should be in the global window since all output will be as well.
+   * A transform evaluator for the pseudo-primitive {@link DirectGroupAlsoByWindow}. The window of
+   * the input {@link KeyedWorkItem} is ignored; it should be in the global window, as element
+   * windows are reified in the {@link KeyedWorkItem#elementsIterable()}.
    *
    * @see GroupByKeyViaGroupByKeyOnly
    */
@@ -173,7 +172,7 @@ public void processElement(WindowedValue<KeyedWorkItem<K, V>> element) throws Ex
               stateInternals,
               timerInternals,
               new OutputWindowedValueToBundle<>(bundle),
-              new UnsupportedSideInputReader("GroupAlsoByWindow"),
+              new UnsupportedSideInputReader(DirectGroupAlsoByWindow.class.getSimpleName()),
               reduceFn,
               evaluationContext.getPipelineOptions());
 
@@ -226,8 +225,9 @@ public boolean apply(WindowedValue<V> input) {
                     // The element is too late for this window.
                     droppedDueToLateness.inc();
                     WindowTracing.debug(
-                        "GroupAlsoByWindow: Dropping element at {} for key: {}; "
+                        "{}: Dropping element at {} for key: {}; "
                             + "window: {} since it is too far behind inputWatermark: {}",
+                        DirectGroupAlsoByWindow.class.getSimpleName(),
                         input.getTimestamp(),
                         key,
                         window,
@@ -264,7 +264,9 @@ public <AdditionalOutputT> void outputWindowedValue(
         Instant timestamp,
         Collection<? extends BoundedWindow> windows,
         PaneInfo pane) {
-      throw new UnsupportedOperationException("GroupAlsoByWindow should not use tagged outputs");
+      throw new UnsupportedOperationException(
+          String.format(
+              "%s should not use tagged outputs", DirectGroupAlsoByWindow.class.getSimpleName()));
     }
   }
 }

From 143334ca532076c839e5699171c8d9db8743b6a2 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 12:04:53 -0700
Subject: [PATCH 274/387] Minor touch-ups to new BoundedWindow javadoc

---
 .../transforms/windowing/BoundedWindow.java   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
index 506753dc4f77..92fa3c59f62c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/BoundedWindow.java
@@ -21,19 +21,19 @@
 import org.joda.time.Instant;
 
 /**
- * A {@code BoundedWindow} represents window information assigned to data elements.
+ * A {@link BoundedWindow} represents window information assigned to data elements.
  *
- * <p>It has one method {@link #maxTimestamp()} to define an upper bound(included) of element
- * timestamp. Elements within the timestamp range are placed in this window.
- * Technically a window may not have a lower bound. Upper bound is mandatory as watermark,
- * which also controls discarding data and expiring the window, is based on event timestamp.
+ * <p>It has one method {@link #maxTimestamp()} to define an upper bound (inclusive) for element
+ * timestamps. A {@link WindowFn} must assign an element only to windows where {@link
+ * #maxTimestamp()} is greater than or equal to the element timestamp. When the watermark passes the
+ * maximum timestamp, all data for a window is estimated to be received.
  *
- * <p>At some point in time, usually when watermark passes,
- * all data with the same window will arrive and can be processed together.
+ * <p>A window does not need to have a lower bound. Only the upper bound is mandatory because it
+ * governs management of triggering and discarding of the window.
  *
- * <p>Windows must also implement {@link Object#equals} and
- * {@link Object#hashCode} such that windows that are logically equal will
- * be treated as equal by {@code equals()} and {@code hashCode()}.
+ * <p>Windows must also implement {@link Object#equals} and {@link Object#hashCode} such that
+ * windows that are logically equal will be treated as equal by {@code equals()} and {@code
+ * hashCode()}.
  */
 public abstract class BoundedWindow {
   // The min and max timestamps that won't overflow when they are converted to

From 10357c20ba62dab873749f135d23c4dda4033cd7 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 19 May 2017 14:06:12 -0700
Subject: [PATCH 275/387] Remove Unused DoFnInfo methods

---
 .../beam/runners/dataflow/util/DoFnInfo.java  | 25 -------------------
 .../control/ProcessBundleHandlerTest.java     |  2 +-
 2 files changed, 1 insertion(+), 26 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
index bd2742f147da..4a26795ec138 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
@@ -53,25 +53,6 @@ public static <InputT, OutputT> DoFnInfo<InputT, OutputT> forFn(
         doFn, windowingStrategy, sideInputViews, inputCoder, mainOutput, outputMap);
   }
 
-  /** TODO: remove this when Dataflow worker uses the DoFn overload. */
-  @Deprecated
-  @SuppressWarnings("unchecked")
-  public static <InputT, OutputT> DoFnInfo<InputT, OutputT> forFn(
-      Serializable doFn,
-      WindowingStrategy<?, ?> windowingStrategy,
-      Iterable<PCollectionView<?>> sideInputViews,
-      Coder<InputT> inputCoder,
-      long mainOutput,
-      Map<Long, TupleTag<?>> outputMap) {
-    return forFn(
-        (DoFn<InputT, OutputT>) doFn,
-        windowingStrategy,
-        sideInputViews,
-        inputCoder,
-        mainOutput,
-        outputMap);
-  }
-
   public DoFnInfo<InputT, OutputT> withFn(DoFn<InputT, OutputT> newFn) {
     return DoFnInfo.forFn(newFn,
         windowingStrategy,
@@ -96,12 +77,6 @@ private DoFnInfo(
     this.outputMap = outputMap;
   }
 
-  /** TODO: remove this when Dataflow worker uses {@link #getDoFn}. */
-  @Deprecated
-  public Serializable getFn() {
-    return doFn;
-  }
-
   /** Returns the embedded function. */
   public DoFn<InputT, OutputT> getDoFn() {
     return doFn;
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
index 748ffea1f13c..f40572843bee 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java
@@ -340,7 +340,7 @@ public void testCreatingAndProcessingDoFn() throws Exception {
         new TestDoFn(),
         WindowingStrategy.globalDefault(),
         ImmutableList.of(),
-        STRING_CODER,
+        StringUtf8Coder.of(),
         mainOutputId,
         ImmutableMap.of(
             mainOutputId, TestDoFn.mainOutput,

From 888a5e6ab9c1c83ee06281de4c906be69c076286 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 19 May 2017 14:24:07 -0700
Subject: [PATCH 276/387] Extract the Main Input PCollection in ParDos

---
 .../beam/runners/core/construction/ParDos.java   | 16 ++++++++++++++++
 .../runners/core/construction/ParDosTest.java    |  4 ++++
 2 files changed, 20 insertions(+)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
index 4752bd1a8940..2ecc04161fed 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
@@ -23,6 +23,8 @@
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
 import com.google.common.base.Optional;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
 import com.google.protobuf.Any;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.BytesValue;
@@ -161,6 +163,20 @@ public static TupleTag<?> getMainOutputTag(ParDoPayload payload)
     return doFnAndMainOutputTagFromProto(payload.getDoFn()).getMainOutputTag();
   }
 
+  public static RunnerApi.PCollection getMainInput(
+      RunnerApi.PTransform ptransform, Components components) throws IOException {
+    checkArgument(
+        ptransform.getSpec().getUrn().equals(PAR_DO_PAYLOAD_URN),
+        "Unexpected payload type %s",
+        ptransform.getSpec().getUrn());
+    ParDoPayload payload = ptransform.getSpec().getParameter().unpack(ParDoPayload.class);
+    String mainInputId =
+        Iterables.getOnlyElement(
+            Sets.difference(
+                ptransform.getInputsMap().keySet(), payload.getSideInputsMap().keySet()));
+    return components.getPcollectionsOrThrow(ptransform.getInputsOrThrow(mainInputId));
+  }
+
   // TODO: Implement
   private static StateSpec toProto(StateDeclaration state) {
     throw new UnsupportedOperationException("Not yet supported");
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java
index 74edec15af81..b6f0b7d9d68e 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java
@@ -149,6 +149,10 @@ public void toAndFromTransformProto() throws Exception {
               view.getWindowingStrategyInternal().fixDefaults()));
       assertThat(restoredView.getCoderInternal(), equalTo(view.getCoderInternal()));
     }
+    String mainInputId = components.registerPCollection(mainInput);
+    assertThat(
+        ParDos.getMainInput(protoTransform, protoComponents),
+        equalTo(protoComponents.getPcollectionsOrThrow(mainInputId)));
   }
 
   private static class DropElementsFn extends DoFn<KV<Long, String>, Void> {

From 43e291015b07fe8f98cb70131bb2482dfa633c7c Mon Sep 17 00:00:00 2001
From: Stephen Sisk <sisk@google.com>
Date: Mon, 22 May 2017 11:36:09 -0700
Subject: [PATCH 277/387] Cleanup k8s scripts naming & don't create insecure
 svc by default

These scripts setup a internet-accessible service by default, which is
insecure since we rely on firewalls for securing the data stores.
---
 .../cassandra/LargeITCluster/{start-up.sh => setup.sh}           | 1 -
 .test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh      | 1 -
 .../cassandra/SmallITCluster/{start-up.sh => setup.sh}           | 1 -
 .test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh      | 1 -
 .../LargeProductionCluster/{start-up.sh => setup.sh}             | 1 -
 .../kubernetes/elasticsearch/LargeProductionCluster/teardown.sh  | 1 -
 .../elasticsearch/SmallITCluster/{start-up.sh => setup.sh}       | 1 -
 .test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh  | 1 -
 8 files changed, 8 deletions(-)
 rename .test-infra/kubernetes/cassandra/LargeITCluster/{start-up.sh => setup.sh} (94%)
 rename .test-infra/kubernetes/cassandra/SmallITCluster/{start-up.sh => setup.sh} (94%)
 rename .test-infra/kubernetes/elasticsearch/LargeProductionCluster/{start-up.sh => setup.sh} (94%)
 rename .test-infra/kubernetes/elasticsearch/SmallITCluster/{start-up.sh => setup.sh} (94%)

diff --git a/.test-infra/kubernetes/cassandra/LargeITCluster/start-up.sh b/.test-infra/kubernetes/cassandra/LargeITCluster/setup.sh
similarity index 94%
rename from .test-infra/kubernetes/cassandra/LargeITCluster/start-up.sh
rename to .test-infra/kubernetes/cassandra/LargeITCluster/setup.sh
index 7341209ccaf8..7bc08099a9ad 100644
--- a/.test-infra/kubernetes/cassandra/LargeITCluster/start-up.sh
+++ b/.test-infra/kubernetes/cassandra/LargeITCluster/setup.sh
@@ -18,5 +18,4 @@
 set -e
 
 # Create Cassandra services and statefulset.
-kubectl create -f cassandra-service-for-local-dev.yaml
 kubectl create -f cassandra-svc-statefulset.yaml
diff --git a/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh b/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh
index 367b6049fcbe..3d040a637a0c 100644
--- a/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh
+++ b/.test-infra/kubernetes/cassandra/LargeITCluster/teardown.sh
@@ -20,6 +20,5 @@ set -e
 
 # Delete Cassandra services and statefulset.
 kubectl delete -f cassandra-svc-statefulset.yaml
-kubectl delete -f cassandra-service-for-local-dev.yaml
 # Delete the persistent storage media for the PersistentVolumes
 kubectl delete pvc -l app=cassandra
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh b/.test-infra/kubernetes/cassandra/SmallITCluster/setup.sh
similarity index 94%
rename from .test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
rename to .test-infra/kubernetes/cassandra/SmallITCluster/setup.sh
index 9377a9c190a0..fad6df05ca24 100644
--- a/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
+++ b/.test-infra/kubernetes/cassandra/SmallITCluster/setup.sh
@@ -18,6 +18,5 @@
 set -e
 
 # Create Cassandra services and Replication controller.
-kubectl create -f cassandra-service-for-local-dev.yaml
 kubectl create -f cassandra-svc-rc.yaml
 
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh b/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
index f4ad0be90456..f538a753eb0a 100644
--- a/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
+++ b/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
@@ -19,4 +19,3 @@ set -e
 
 # Delete Cassandra services and Replication controller.
 kubectl delete -f cassandra-svc-rc.yaml
-kubectl delete -f cassandra-service-for-local-dev.yaml
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/setup.sh
similarity index 94%
rename from .test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
rename to .test-infra/kubernetes/elasticsearch/LargeProductionCluster/setup.sh
index 93022c724400..9fbb6c31ad57 100644
--- a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
+++ b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/setup.sh
@@ -18,5 +18,4 @@
 set -e
 
 # Create Elasticsearch services and deployments.
-kubectl create -f elasticsearch-service-for-local-dev.yaml
 kubectl create -f es-services-deployments.yaml
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
index bdc9ab9341a6..18568a3e4230 100644
--- a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
+++ b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
@@ -18,4 +18,3 @@ set -e
 
 # Delete elasticsearch services and deployments.
 kubectl delete -f es-services-deployments.yaml
-kubectl delete -f elasticsearch-service-for-local-dev.yaml
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh b/.test-infra/kubernetes/elasticsearch/SmallITCluster/setup.sh
similarity index 94%
rename from .test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
rename to .test-infra/kubernetes/elasticsearch/SmallITCluster/setup.sh
index 2d6522ea9c54..e8cf275d9729 100644
--- a/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
+++ b/.test-infra/kubernetes/elasticsearch/SmallITCluster/setup.sh
@@ -18,6 +18,5 @@
 set -e
 
 # Create Elasticsearch services and deployments.
-kubectl create -f elasticsearch-service-for-local-dev.yaml
 kubectl create -f elasticsearch-svc-rc.yaml
 
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh b/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
index 61c079fed894..079141d8cd9b 100644
--- a/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
+++ b/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
@@ -18,4 +18,3 @@ set -e
 
 # Delete elasticsearch services and deployments.
 kubectl delete -f elasticsearch-svc-rc.yaml
-kubectl delete -f elasticsearch-service-for-local-dev.yaml

From 13d20122bc0b20e1348714409fbc3b6b1f1c1e4f Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 22 May 2017 13:00:52 -0700
Subject: [PATCH 278/387] [BEAM-1999] Remove resolved todo from value providers

---
 sdks/python/apache_beam/options/value_provider.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sdks/python/apache_beam/options/value_provider.py b/sdks/python/apache_beam/options/value_provider.py
index 40bddba5a19c..f7630bd8d326 100644
--- a/sdks/python/apache_beam/options/value_provider.py
+++ b/sdks/python/apache_beam/options/value_provider.py
@@ -82,7 +82,6 @@ def get(self):
       value = self.default_value
     return value
 
-  # TODO(BEAM-1999): Remove _unused_options_id
   @classmethod
   def set_runtime_options(cls, pipeline_options):
     RuntimeValueProvider.runtime_options = pipeline_options

From 39e8d37ee95709d07a74c9df866059871d5585f5 Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Mon, 22 May 2017 13:03:38 -0700
Subject: [PATCH 279/387] Update Dataflow BATCH_ENVIRONMENT_MAJOR_VERSION to 6

---
 sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 0ecd22ac80b2..a05e582b05f4 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -62,7 +62,7 @@ class DataflowRunner(PipelineRunner):
   # Environment version information. It is passed to the service during a
   # a job submission and is used by the service to establish what features
   # are expected by the workers.
-  BATCH_ENVIRONMENT_MAJOR_VERSION = '5'
+  BATCH_ENVIRONMENT_MAJOR_VERSION = '6'
   STREAMING_ENVIRONMENT_MAJOR_VERSION = '0'
 
   def __init__(self, cache=None):

From 7f35c98b5f77069bd21dc7dea4a7d046883d13a6 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 18 May 2017 10:23:35 -0700
Subject: [PATCH 280/387] Add ReadTranslator

This translates Read transforms to ReadPayloads and back
---
 .../core/construction/ReadTranslator.java     | 127 +++++++++++++
 .../core/construction/ReadTranslatorTest.java | 179 ++++++++++++++++++
 .../beam/runners/dataflow/ReadTranslator.java |   5 +-
 .../apache/beam/sdk/io/CountingSource.java    |  42 ++++
 4 files changed, 350 insertions(+), 3 deletions(-)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslator.java
 create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslatorTest.java

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslator.java
new file mode 100644
index 000000000000..f94493862b2e
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslator.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.protobuf.Any;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.BytesValue;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.IsBounded;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.ReadPayload;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.SerializableUtils;
+
+/**
+ * Methods for translating {@link Read.Bounded} and {@link Read.Unbounded}
+ * {@link PTransform PTransforms} into {@link ReadPayload} protos.
+ */
+public class ReadTranslator {
+  private static final String JAVA_SERIALIZED_BOUNDED_SOURCE = "urn:beam:java:boundedsource:v1";
+  private static final String JAVA_SERIALIZED_UNBOUNDED_SOURCE = "urn:beam:java:unboundedsource:v1";
+
+  public static ReadPayload toProto(Read.Bounded<?> read) {
+    return ReadPayload.newBuilder()
+        .setIsBounded(IsBounded.BOUNDED)
+        .setSource(toProto(read.getSource()))
+        .build();
+  }
+
+  public static ReadPayload toProto(Read.Unbounded<?> read) {
+    return ReadPayload.newBuilder()
+        .setIsBounded(IsBounded.UNBOUNDED)
+        .setSource(toProto(read.getSource()))
+        .build();
+  }
+
+  public static SdkFunctionSpec toProto(Source<?> source) {
+    if (source instanceof BoundedSource) {
+      return toProto((BoundedSource) source);
+    } else if (source instanceof UnboundedSource) {
+      return toProto((UnboundedSource<?, ?>) source);
+    } else {
+      throw new IllegalArgumentException(
+          String.format("Unknown %s type %s", Source.class.getSimpleName(), source.getClass()));
+    }
+  }
+
+  private static SdkFunctionSpec toProto(BoundedSource<?> source) {
+    return SdkFunctionSpec.newBuilder()
+        .setSpec(
+            FunctionSpec.newBuilder()
+                .setUrn(JAVA_SERIALIZED_BOUNDED_SOURCE)
+                .setParameter(
+                    Any.pack(
+                        BytesValue.newBuilder()
+                            .setValue(
+                                ByteString.copyFrom(SerializableUtils.serializeToByteArray(source)))
+                            .build())))
+        .build();
+  }
+
+  public static BoundedSource<?> boundedSourceFromProto(ReadPayload payload)
+      throws InvalidProtocolBufferException {
+    checkArgument(payload.getIsBounded().equals(IsBounded.BOUNDED));
+    return (BoundedSource<?>) SerializableUtils.deserializeFromByteArray(
+        payload
+            .getSource()
+            .getSpec()
+            .getParameter()
+            .unpack(BytesValue.class)
+            .getValue()
+            .toByteArray(),
+        "BoundedSource");
+  }
+
+  private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source) {
+    return SdkFunctionSpec.newBuilder()
+        .setSpec(
+            FunctionSpec.newBuilder()
+                .setUrn(JAVA_SERIALIZED_UNBOUNDED_SOURCE)
+                .setParameter(
+                    Any.pack(
+                        BytesValue.newBuilder()
+                            .setValue(
+                                ByteString.copyFrom(SerializableUtils.serializeToByteArray(source)))
+                            .build())))
+        .build();
+  }
+
+  public static UnboundedSource<?, ?> unboundedSourceFromProto(ReadPayload payload)
+      throws InvalidProtocolBufferException {
+    checkArgument(payload.getIsBounded().equals(IsBounded.UNBOUNDED));
+    return (UnboundedSource<?, ?>) SerializableUtils.deserializeFromByteArray(
+        payload
+            .getSource()
+            .getSpec()
+            .getParameter()
+            .unpack(BytesValue.class)
+            .getValue()
+            .toByteArray(),
+        "BoundedSource");
+  }
+
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslatorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslatorTest.java
new file mode 100644
index 000000000000..a603e342bc9f
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslatorTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assume.assumeThat;
+
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.ReadPayload;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.CountingSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests for {@link ReadTranslator}.
+ */
+@RunWith(Parameterized.class)
+public class ReadTranslatorTest {
+
+  @Parameters(name = "{index}: {0}")
+  public static Iterable<Source<?>> data() {
+    return ImmutableList.<Source<?>>of(
+        CountingSource.unbounded(),
+        CountingSource.upTo(100L),
+        new TestBoundedSource(),
+        new TestUnboundedSource());
+  }
+
+  @Parameter(0)
+  public Source<?> source;
+
+  @Test
+  public void testToFromProtoBounded() throws Exception {
+    // TODO: Split into two tests.
+    assumeThat(source, instanceOf(BoundedSource.class));
+    BoundedSource<?> boundedSource = (BoundedSource<?>) this.source;
+    Read.Bounded<?> boundedRead = Read.from(boundedSource);
+    ReadPayload payload = ReadTranslator.toProto(boundedRead);
+    assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.BOUNDED));
+    BoundedSource<?> deserializedSource = ReadTranslator.boundedSourceFromProto(payload);
+    assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
+  }
+
+  @Test
+  public void testToFromProtoUnbounded() throws Exception {
+    assumeThat(source, instanceOf(UnboundedSource.class));
+    UnboundedSource<?, ?> unboundedSource = (UnboundedSource<?, ?>) this.source;
+    Read.Unbounded<?> unboundedRead = Read.from(unboundedSource);
+    ReadPayload payload = ReadTranslator.toProto(unboundedRead);
+    assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.UNBOUNDED));
+    UnboundedSource<?, ?> deserializedSource = ReadTranslator.unboundedSourceFromProto(payload);
+    assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
+  }
+
+  private static class TestBoundedSource extends BoundedSource<String> {
+    @Override
+    public List<? extends BoundedSource<String>> split(
+        long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public BoundedReader<String> createReader(PipelineOptions options) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void validate() {}
+
+    @Override
+    public Coder<String> getDefaultOutputCoder() {
+      return StringUtf8Coder.of();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && other.getClass().equals(TestBoundedSource.class);
+    }
+
+    @Override
+    public int hashCode() {
+      return TestBoundedSource.class.hashCode();
+    }
+  }
+
+  private static class TestUnboundedSource extends UnboundedSource<byte[], CheckpointMark> {
+    @Override
+    public void validate() {}
+
+    @Override
+    public Coder<byte[]> getDefaultOutputCoder() {
+      return ByteArrayCoder.of();
+    }
+
+    @Override
+    public List<? extends UnboundedSource<byte[], CheckpointMark>> split(
+        int desiredNumSplits, PipelineOptions options) throws Exception {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public UnboundedReader<byte[]> createReader(
+        PipelineOptions options, @Nullable CheckpointMark checkpointMark) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Coder<CheckpointMark> getCheckpointMarkCoder() {
+      return new TestCheckpointMarkCoder();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && other.getClass().equals(TestUnboundedSource.class);
+    }
+
+    @Override
+    public int hashCode() {
+      return TestUnboundedSource.class.hashCode();
+    }
+
+    private class TestCheckpointMarkCoder extends AtomicCoder<CheckpointMark> {
+      @Override
+      public void encode(CheckpointMark value, OutputStream outStream)
+          throws CoderException, IOException {
+        throw new UnsupportedOperationException();
+      }
+
+      @Override
+      public CheckpointMark decode(InputStream inStream) throws CoderException, IOException {
+        throw new UnsupportedOperationException();
+      }
+    }
+  }
+}
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
index 30ecbf581d2a..0b22d7e8ceed 100755
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/ReadTranslator.java
@@ -40,9 +40,8 @@ public void translate(Read.Bounded<?> transform, TranslationContext context) {
     translateReadHelper(transform.getSource(), transform, context);
   }
 
-  public static <T> void translateReadHelper(Source<T> source,
-      PTransform<?, ? extends PValue> transform,
-      TranslationContext context) {
+  public static <T> void translateReadHelper(
+      Source<T> source, PTransform<?, ? extends PValue> transform, TranslationContext context) {
     try {
       StepTranslationContext stepContext = context.addStep(transform, "ParallelRead");
       stepContext.addInput(PropertyNames.FORMAT, PropertyNames.CUSTOM_SOURCE_FORMAT);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java
index 81082e547f77..6202c2b5fad5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java
@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.NoSuchElementException;
+import java.util.Objects;
 import org.apache.beam.sdk.coders.AvroCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.DefaultCoder;
@@ -136,6 +137,16 @@ static class NowTimestampFn implements SerializableFunction<Long, Instant> {
     public Instant apply(Long input) {
       return Instant.now();
     }
+
+    @Override
+    public boolean equals(Object other) {
+      return other instanceof NowTimestampFn;
+    }
+
+    @Override
+    public int hashCode() {
+      return getClass().hashCode();
+    }
   }
 
   /**
@@ -180,6 +191,21 @@ public org.apache.beam.sdk.io.BoundedSource.BoundedReader<Long> createReader(
     public Coder<Long> getDefaultOutputCoder() {
       return VarLongCoder.of();
     }
+
+    @Override
+    public boolean equals(Object other) {
+      if (!(other instanceof BoundedCountingSource)) {
+        return false;
+      }
+      BoundedCountingSource that = (BoundedCountingSource) other;
+      return this.getStartOffset() == that.getStartOffset()
+          && this.getEndOffset() == that.getEndOffset();
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(this.getStartOffset(), (int) this.getEndOffset());
+    }
   }
 
   /**
@@ -341,6 +367,22 @@ public void validate() {}
     public Coder<Long> getDefaultOutputCoder() {
       return VarLongCoder.of();
     }
+
+    public boolean equals(Object other) {
+      if (!(other instanceof UnboundedCountingSource)) {
+        return false;
+      }
+      UnboundedCountingSource that = (UnboundedCountingSource) other;
+      return this.start == that.start
+          && this.stride == that.stride
+          && this.elementsPerPeriod == that.elementsPerPeriod
+          && Objects.equals(this.period, that.period)
+          && Objects.equals(this.timestampFn, that.timestampFn);
+    }
+
+    public int hashCode() {
+      return Objects.hash(start, stride, elementsPerPeriod, period, timestampFn);
+    }
   }
 
   /**

From 9336230d2a5c18bae89908bcd60db8ea96b7906d Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 4 Apr 2017 17:43:48 -0700
Subject: [PATCH 281/387] Include Additional PTransform inputs in Transform
 Nodes

Add the value of PTransform.getAdditionalInputs in the inputs of a
TransformHierarchy node.

Fork the Node constructor to reduce nullability

This slightly simplifies the constructor implementation(s).

Update the DirectRunner to track main inputs instead of all inputs.
---
 .../apex/translation/TranslationContext.java  |   4 +-
 .../core/construction/TransformInputs.java    |  50 ++++++
 .../construction/TransformInputsTest.java     | 166 ++++++++++++++++++
 .../runners/direct/DirectGraphVisitor.java    |  15 +-
 .../runners/direct/ParDoEvaluatorFactory.java |   9 +-
 ...ttableProcessElementsEvaluatorFactory.java |   2 +
 .../direct/StatefulParDoEvaluatorFactory.java |   1 +
 .../beam/runners/direct/WatermarkManager.java |  17 +-
 .../runners/direct/ParDoEvaluatorTest.java    |   6 +-
 .../flink/FlinkBatchTranslationContext.java   |   3 +-
 .../FlinkStreamingTranslationContext.java     |   3 +-
 .../spark/translation/EvaluationContext.java  |   4 +-
 .../beam/sdk/runners/TransformHierarchy.java  |  28 ++-
 13 files changed, 280 insertions(+), 28 deletions(-)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
 create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index aff3863624c4..94d13e177dec 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -34,6 +34,7 @@
 import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -93,7 +94,8 @@ public Map<TupleTag<?>, PValue> getInputs() {
   }
 
   public <InputT extends PValue> InputT getInput() {
-    return (InputT) Iterables.getOnlyElement(getCurrentTransform().getInputs().values());
+    return (InputT)
+        Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
   }
 
   public Map<TupleTag<?>, PValue> getOutputs() {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
new file mode 100644
index 000000000000..2baf93a3c128
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.ImmutableList;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Utilities for extracting subsets of inputs from an {@link AppliedPTransform}. */
+public class TransformInputs {
+  /**
+   * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link
+   * PTransform#getAdditionalInputs()}.
+   */
+  public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) {
+    ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder();
+    PTransform<?, ?> transform = application.getTransform();
+    for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) {
+      if (!transform.getAdditionalInputs().containsKey(input.getKey())) {
+        mainInputs.add(input.getValue());
+      }
+    }
+    checkArgument(
+        !mainInputs.build().isEmpty() || application.getInputs().isEmpty(),
+        "Expected at least one main input if any inputs exist");
+    return mainInputs.build();
+  }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
new file mode 100644
index 000000000000..f5b2c11e7923
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.junit.Assert.assertThat;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.hamcrest.Matchers;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link TransformInputs}. */
+@RunWith(JUnit4.class)
+public class TransformInputsTest {
+  @Rule public TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+  @Rule public ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  public void nonAdditionalInputsWithNoInputSucceeds() {
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "input-free",
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(),
+            pipeline);
+
+    assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>empty());
+  }
+
+  @Test
+  public void nonAdditionalInputsWithOneMainInputSucceeds() {
+    PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L));
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "input-single",
+            Collections.<TupleTag<?>, PValue>singletonMap(new TupleTag<Long>() {}, input),
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(),
+            pipeline);
+
+    assertThat(
+        TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>containsInAnyOrder(input));
+  }
+
+  @Test
+  public void nonAdditionalInputsWithMultipleNonAdditionalInputsSucceeds() {
+    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+    allInputs.put(new TupleTag<Integer>() {}, mainInts);
+    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+    allInputs.put(new TupleTag<Void>() {}, voids);
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "additional-free",
+            allInputs,
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(),
+            pipeline);
+
+    assertThat(
+        TransformInputs.nonAdditionalInputs(transform),
+        Matchers.<PValue>containsInAnyOrder(voids, mainInts));
+  }
+
+  @Test
+  public void nonAdditionalInputsWithAdditionalInputsSucceeds() {
+    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+    allInputs.put(new TupleTag<Integer>() {}, mainInts);
+    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+    allInputs.put(
+        new TupleTag<Void>() {}, voids);
+    allInputs.putAll(additionalInputs);
+
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "additional",
+            allInputs,
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(additionalInputs),
+            pipeline);
+
+    assertThat(
+        TransformInputs.nonAdditionalInputs(transform),
+        Matchers.<PValue>containsInAnyOrder(mainInts, voids));
+  }
+
+  @Test
+  public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() {
+    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "additional-only",
+            additionalInputs,
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(additionalInputs),
+            pipeline);
+
+    thrown.expect(IllegalArgumentException.class);
+    thrown.expectMessage("at least one");
+    TransformInputs.nonAdditionalInputs(transform);
+  }
+
+  private static class TestTransform extends PTransform<PInput, POutput> {
+    private final Map<TupleTag<?>, PValue> additionalInputs;
+
+    private TestTransform() {
+      this(Collections.<TupleTag<?>, PValue>emptyMap());
+    }
+
+    private TestTransform(Map<TupleTag<?>, PValue> additionalInputs) {
+      this.additionalInputs = additionalInputs;
+    }
+
+    @Override
+    public POutput expand(PInput input) {
+      return PDone.in(input.getPipeline());
+    }
+
+    @Override
+    public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+      return additionalInputs;
+    }
+  }
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index 01204e3049dd..ed4282bbdb8b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -21,10 +21,12 @@
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -34,6 +36,8 @@
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Tracks the {@link AppliedPTransform AppliedPTransforms} that consume each {@link PValue} in the
@@ -41,6 +45,7 @@
  * input after the upstream transform has produced and committed output.
  */
 class DirectGraphVisitor extends PipelineVisitor.Defaults {
+  private static final Logger LOG = LoggerFactory.getLogger(DirectGraphVisitor.class);
 
   private Map<POutput, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
 
@@ -83,7 +88,15 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
     if (node.getInputs().isEmpty()) {
       rootTransforms.add(appliedTransform);
     } else {
-      for (PValue value : node.getInputs().values()) {
+      Collection<PValue> mainInputs =
+          TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline()));
+      if (!mainInputs.containsAll(node.getInputs().values())) {
+        LOG.debug(
+            "Inputs reduced to {} from {} by removing additional inputs",
+            mainInputs,
+            node.getInputs().values());
+      }
+      for (PValue value : mainInputs) {
         primitiveConsumers.put(value, appliedTransform);
       }
     }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 74470bfb8b8d..c52091e27cfe 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -20,7 +20,6 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -79,6 +78,7 @@ public <T> TransformEvaluator<T> forApplication(
         (TransformEvaluator<T>)
             createEvaluator(
                 (AppliedPTransform) application,
+                (PCollection<InputT>) inputBundle.getPCollection(),
                 inputBundle.getKey(),
                 doFn,
                 transform.getSideInputs(),
@@ -102,6 +102,7 @@ public void cleanup() throws Exception {
   @SuppressWarnings({"unchecked", "rawtypes"})
   DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
+      PCollection<InputT> mainInput,
       StructuralKey<?> inputBundleKey,
       DoFn<InputT, OutputT> doFn,
       List<PCollectionView<?>> sideInputs,
@@ -120,6 +121,7 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
         createParDoEvaluator(
             application,
             inputBundleKey,
+            mainInput,
             sideInputs,
             mainOutputTag,
             additionalOutputTags,
@@ -132,6 +134,7 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
   ParDoEvaluator<InputT> createParDoEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
       StructuralKey<?> key,
+      PCollection<InputT> mainInput,
       List<PCollectionView<?>> sideInputs,
       TupleTag<OutputT> mainOutputTag,
       List<TupleTag<?>> additionalOutputTags,
@@ -144,8 +147,7 @@ ParDoEvaluator<InputT> createParDoEvaluator(
           evaluationContext,
           stepContext,
           application,
-          ((PCollection<InputT>) Iterables.getOnlyElement(application.getInputs().values()))
-              .getWindowingStrategy(),
+          mainInput.getWindowingStrategy(),
           fn,
           key,
           sideInputs,
@@ -173,5 +175,4 @@ static Map<TupleTag<?>, PCollection<?>> pcollections(Map<TupleTag<?>, PValue> ou
     }
     return pcs;
   }
-
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index dc85d87bc93f..4e7f4db65478 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -116,6 +116,8 @@ public void cleanup() throws Exception {
             delegateFactory.createParDoEvaluator(
                 application,
                 inputBundle.getKey(),
+                (PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>)
+                    inputBundle.getPCollection(),
                 transform.getSideInputs(),
                 transform.getMainOutputTag(),
                 transform.getAdditionalOutputTags().getAll(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 985c3be4e9e9..e22edd187c53 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -117,6 +117,7 @@ private TransformEvaluator<KeyedWorkItem<K, KV<K, InputT>>> createEvaluator(
     DoFnLifecycleManagerRemovingTransformEvaluator<KV<K, InputT>> delegateEvaluator =
         delegateFactory.createEvaluator(
             (AppliedPTransform) application,
+            (PCollection) inputBundle.getPCollection(),
             inputBundle.getKey(),
             doFn,
             application.getTransform().getUnderlyingParDo().getSideInputs(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index 4f1b8319dc2d..b15b52e314de 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -823,10 +823,11 @@ private Collection<Watermark> getInputProcessingWatermarks(AppliedPTransform<?,
       inputWmsBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      Watermark producerOutputWatermark =
-          getTransformWatermark(graph.getProducer(pvalue))
-              .synchronizedProcessingOutputWatermark;
-      inputWmsBuilder.add(producerOutputWatermark);
+      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
+        Watermark producerOutputWatermark =
+            getTransformWatermark(graph.getProducer(pvalue)).synchronizedProcessingOutputWatermark;
+        inputWmsBuilder.add(producerOutputWatermark);
+      }
     }
     return inputWmsBuilder.build();
   }
@@ -838,9 +839,11 @@ private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform)
       inputWatermarksBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      Watermark producerOutputWatermark =
-          getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
-      inputWatermarksBuilder.add(producerOutputWatermark);
+      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
+        Watermark producerOutputWatermark =
+            getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
+        inputWatermarksBuilder.add(producerOutputWatermark);
+      }
     }
     List<Watermark> inputCollectionWatermarks = inputWatermarksBuilder.build();
     return inputCollectionWatermarks;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 286e44d1be04..3b2a22ee26f9 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -98,7 +98,7 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
     when(evaluationContext.createBundle(output)).thenReturn(outputBundle);
 
     ParDoEvaluator<Integer> evaluator =
-        createEvaluator(singletonView, fn, output);
+        createEvaluator(singletonView, fn, inputPc, output);
 
     IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
     WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
@@ -132,6 +132,7 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
   private ParDoEvaluator<Integer> createEvaluator(
       PCollectionView<Integer> singletonView,
       RecorderFn fn,
+      PCollection<Integer> input,
       PCollection<Integer> output) {
     when(
             evaluationContext.createSideInputReader(
@@ -156,8 +157,7 @@ private ParDoEvaluator<Integer> createEvaluator(
         evaluationContext,
         stepContext,
         transform,
-        ((PCollection<?>) Iterables.getOnlyElement(transform.getInputs().values()))
-            .getWindowingStrategy(),
+        input.getWindowingStrategy(),
         fn,
         null /* key */,
         ImmutableList.<PCollectionView<?>>of(singletonView),
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index 0439119dfc40..6e7019848b19 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -20,6 +20,7 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -143,7 +144,7 @@ Map<TupleTag<?>, PValue> getInputs(PTransform<?, ?> transform) {
 
   @SuppressWarnings("unchecked")
   <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
   }
 
   Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
index ea5f6b3162af..74a5fb971144 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
@@ -22,6 +22,7 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -113,7 +114,7 @@ public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collecti
 
   @SuppressWarnings("unchecked")
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
   }
 
   public <T extends PInput> Map<TupleTag<?>, PValue> getInputs(PTransform<T, ?> transform) {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 8102926f6daa..0c6c4d1cb660 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -26,6 +26,7 @@
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.sdk.Pipeline;
@@ -103,7 +104,8 @@ public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
 
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
     @SuppressWarnings("unchecked")
-    T input = (T) Iterables.getOnlyElement(getInputs(transform).values());
+    T input =
+        (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
     return input;
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 2f0e8efd7de8..9d73b4576788 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -32,7 +32,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
@@ -68,7 +67,7 @@ public TransformHierarchy() {
     producers = new HashMap<>();
     producerInput = new HashMap<>();
     unexpandedInputs = new HashMap<>();
-    root = new Node(null, null, "", null);
+    root = new Node();
     current = root;
   }
 
@@ -252,26 +251,37 @@ public class Node {
     @VisibleForTesting
     boolean finishedSpecifying = false;
 
+    /**
+     * Creates the root-level node. The root level node has a null enclosing node, a null transform,
+     * an empty map of inputs, and a name equal to the empty string.
+     */
+    private Node() {
+      this.enclosingNode = null;
+      this.transform = null;
+      this.fullName = "";
+      this.inputs = Collections.emptyMap();
+    }
+
     /**
      * Creates a new Node with the given parent and transform.
      *
-     * <p>EnclosingNode and transform may both be null for a root-level node, which holds all other
-     * nodes.
-     *
      * @param enclosingNode the composite node containing this node
      * @param transform the PTransform tracked by this node
      * @param fullName the fully qualified name of the transform
      * @param input the unexpanded input to the transform
      */
     private Node(
-        @Nullable Node enclosingNode,
-        @Nullable PTransform<?, ?> transform,
+        Node enclosingNode,
+        PTransform<?, ?> transform,
         String fullName,
-        @Nullable PInput input) {
+        PInput input) {
       this.enclosingNode = enclosingNode;
       this.transform = transform;
       this.fullName = fullName;
-      this.inputs = input == null ? Collections.<TupleTag<?>, PValue>emptyMap() : input.expand();
+      ImmutableMap.Builder<TupleTag<?>, PValue> inputs = ImmutableMap.builder();
+      inputs.putAll(input.expand());
+      inputs.putAll(transform.getAdditionalInputs());
+      this.inputs = inputs.build();
     }
 
     /**

From bb3db2acc07a61844a551ea9f793a0fcf4001878 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Wed, 17 May 2017 15:08:50 -0700
Subject: [PATCH 282/387] Add WindowIntoTranslator

This translates Window.Assign into a WindowIntoPayload.
---
 .../construction/WindowIntoTranslator.java    |  61 +++++++++
 .../construction/WindowingStrategies.java     |  33 ++---
 .../WindowIntoTranslatorTest.java             | 126 ++++++++++++++++++
 3 files changed, 205 insertions(+), 15 deletions(-)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
 create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
new file mode 100644
index 000000000000..ea4c9965790c
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.google.protobuf.Any;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.beam.runners.core.construction.PTransforms.TransformPayloadTranslator;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.WindowIntoPayload;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+
+/**
+ * Utility methods for translating a {@link Window.Assign} to and from {@link RunnerApi}
+ * representations.
+ */
+public class WindowIntoTranslator {
+
+  static class WindowAssignTranslator implements TransformPayloadTranslator<Window.Assign<?>> {
+    @Override
+    public FunctionSpec translate(
+        AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) {
+      return FunctionSpec.newBuilder()
+          .setUrn("urn:beam:transform:window:v1")
+          .setParameter(
+              Any.pack(WindowIntoTranslator.toProto(transform.getTransform(), components)))
+          .build();
+    }
+  }
+
+  public static WindowIntoPayload toProto(Window.Assign<?> transform, SdkComponents components) {
+    return WindowIntoPayload.newBuilder()
+        .setWindowFn(WindowingStrategies.toProto(transform.getWindowFn(), components))
+        .build();
+  }
+
+  public static WindowFn<?, ?> getWindowFn(WindowIntoPayload payload)
+      throws InvalidProtocolBufferException {
+    SdkFunctionSpec spec = payload.getWindowFn();
+    return WindowingStrategies.windowFnFromProto(spec);
+  }
+}
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
index 395702f51a3e..8dceebbe6cc3 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
@@ -163,8 +163,7 @@ public static TimestampCombiner timestampCombinerFromProto(RunnerApi.OutputTime
    * input {@link WindowFn}.
    */
   public static SdkFunctionSpec toProto(
-      WindowFn<?, ?> windowFn, @SuppressWarnings("unused") SdkComponents components)
-      throws IOException {
+      WindowFn<?, ?> windowFn, @SuppressWarnings("unused") SdkComponents components) {
     return SdkFunctionSpec.newBuilder()
         // TODO: Set environment ID
         .setSpec(
@@ -245,7 +244,23 @@ public static RunnerApi.WindowingStrategy toProto(
       throws InvalidProtocolBufferException {
 
     SdkFunctionSpec windowFnSpec = proto.getWindowFn();
+    WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec);
+    TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
+    AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
+    Trigger trigger = Triggers.fromProto(proto.getTrigger());
+    ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
+    Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
+
+    return WindowingStrategy.of(windowFn)
+        .withAllowedLateness(allowedLateness)
+        .withMode(accumulationMode)
+        .withTrigger(trigger)
+        .withTimestampCombiner(timestampCombiner)
+        .withClosingBehavior(closingBehavior);
+  }
 
+  public static WindowFn<?, ?> windowFnFromProto(SdkFunctionSpec windowFnSpec)
+      throws InvalidProtocolBufferException {
     checkArgument(
         windowFnSpec.getSpec().getUrn().equals(CUSTOM_WINDOWFN_URN),
         "Only Java-serialized %s instances are supported, with URN %s. But found URN %s",
@@ -258,18 +273,6 @@ public static RunnerApi.WindowingStrategy toProto(
             windowFnSpec.getSpec().getParameter().unpack(BytesValue.class).getValue().toByteArray(),
             "WindowFn");
 
-    WindowFn<?, ?> windowFn = (WindowFn<?, ?>) deserializedWindowFn;
-    TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
-    AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
-    Trigger trigger = Triggers.fromProto(proto.getTrigger());
-    ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
-    Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
-
-    return WindowingStrategy.of(windowFn)
-        .withAllowedLateness(allowedLateness)
-        .withMode(accumulationMode)
-        .withTrigger(trigger)
-        .withTimestampCombiner(timestampCombiner)
-        .withClosingBehavior(closingBehavior);
+    return (WindowFn<?, ?>) deserializedWindowFn;
   }
 }
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java
new file mode 100644
index 000000000000..fbac56516cc1
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkState;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.WindowIntoPayload;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.PartitioningWindowFn;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.Window.Assign;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests for {@link WindowIntoTranslator}.
+ */
+@RunWith(Parameterized.class)
+public class WindowIntoTranslatorTest {
+  @Parameters(name = "{index}: {0}")
+  public static Iterable<WindowFn<?, ?>> data() {
+    // This pipeline exists for construction, not to run any test.
+    return ImmutableList.<WindowFn<?, ?>>builder()
+        .add(FixedWindows.of(Duration.standardMinutes(10L)))
+        .add(new GlobalWindows())
+        .add(Sessions.withGapDuration(Duration.standardMinutes(15L)))
+        .add(SlidingWindows.of(Duration.standardMinutes(5L)).every(Duration.standardMinutes(1L)))
+        .add(new CustomWindows())
+        .build();
+  }
+
+  @Parameter(0)
+  public WindowFn<?, ?> windowFn;
+
+  @Rule
+  public TestPipeline pipeline = TestPipeline.create();
+
+  @Test
+  public void testToFromProto() throws InvalidProtocolBufferException {
+    pipeline.apply(GenerateSequence.from(0)).apply(Window.<Long>into((WindowFn) windowFn));
+
+    final AtomicReference<AppliedPTransform<?, ?, Assign<?>>> assign = new AtomicReference<>(null);
+    pipeline.traverseTopologically(
+        new PipelineVisitor.Defaults() {
+          @Override
+          public void visitPrimitiveTransform(Node node) {
+            if (node.getTransform() instanceof Window.Assign) {
+              checkState(assign.get() == null);
+              assign.set((AppliedPTransform<?, ?, Assign<?>>) node.toAppliedPTransform());
+            }
+          }
+        });
+    checkState(assign.get() != null);
+
+    SdkComponents components = SdkComponents.create();
+    WindowIntoPayload payload =
+        WindowIntoTranslator.toProto(assign.get().getTransform(), components);
+
+    assertEquals(windowFn, WindowIntoTranslator.getWindowFn(payload));
+  }
+
+  private static class CustomWindows extends PartitioningWindowFn<String, BoundedWindow> {
+    @Override
+    public BoundedWindow assignWindow(Instant timestamp) {
+      return GlobalWindow.INSTANCE;
+    }
+
+    @Override
+    public boolean isCompatible(WindowFn<?, ?> other) {
+      return getClass().equals(other.getClass());
+    }
+
+    @Override
+    public Coder<BoundedWindow> windowCoder() {
+      return (Coder) GlobalWindow.Coder.INSTANCE;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && other.getClass().equals(this.getClass());
+    }
+
+    @Override
+    public int hashCode() {
+      return getClass().hashCode();
+    }
+  }
+}

From 9da46fd053ef3e7538733d45cacad435c312e51b Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 19:02:56 -0700
Subject: [PATCH 283/387] Fix compile error from bad merge

---
 .../runners/core/construction/WindowIntoTranslatorTest.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java
index fbac56516cc1..eaefe2eade3b 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java
@@ -84,7 +84,8 @@ public void testToFromProto() throws InvalidProtocolBufferException {
           public void visitPrimitiveTransform(Node node) {
             if (node.getTransform() instanceof Window.Assign) {
               checkState(assign.get() == null);
-              assign.set((AppliedPTransform<?, ?, Assign<?>>) node.toAppliedPTransform());
+              assign.set(
+                  (AppliedPTransform<?, ?, Assign<?>>) node.toAppliedPTransform(getPipeline()));
             }
           }
         });

From 3c0f599d64a7f57608f1c18b05f2ab036a8b02fc Mon Sep 17 00:00:00 2001
From: Colin Phipps <fipsy@google.com>
Date: Wed, 10 May 2017 09:50:56 +0000
Subject: [PATCH 284/387] Comply with byte limit for Datastore Commit.

---
 .../io/gcp/datastore/v1/datastoreio.py          | 15 ++++++++++++++-
 .../io/gcp/datastore/v1/datastoreio_test.py     | 17 +++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
index c606133b1ece..89c2a93d2780 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
@@ -313,8 +313,12 @@ class _Mutate(PTransform):
   supported, as the commits are retried when failures occur.
   """
 
-  # Max allowed Datastore write batch size.
+  # Max allowed Datastore writes per batch, and max bytes per batch.
+  # Note that the max bytes per batch set here is lower than the 10MB limit
+  # actually enforced by the API, to leave space for the CommitRequest wrapper
+  # around the mutations.
   _WRITE_BATCH_SIZE = 500
+  _WRITE_BATCH_BYTES_SIZE = 9000000
 
   def __init__(self, project, mutation_fn):
     """Initializes a Mutate transform.
@@ -353,13 +357,20 @@ def __init__(self, project):
       self._project = project
       self._datastore = None
       self._mutations = []
+      self._mutations_size = 0  # Total size of entries in _mutations.
 
     def start_bundle(self):
       self._mutations = []
+      self._mutations_size = 0
       self._datastore = helper.get_datastore(self._project)
 
     def process(self, element):
+      size = element.ByteSize()
+      if (self._mutations and
+          size + self._mutations_size > _Mutate._WRITE_BATCH_BYTES_SIZE):
+        self._flush_batch()
       self._mutations.append(element)
+      self._mutations_size += size
       if len(self._mutations) >= _Mutate._WRITE_BATCH_SIZE:
         self._flush_batch()
 
@@ -367,12 +378,14 @@ def finish_bundle(self):
       if self._mutations:
         self._flush_batch()
       self._mutations = []
+      self._mutations_size = 0
 
     def _flush_batch(self):
       # Flush the current batch of mutations to Cloud Datastore.
       helper.write_mutations(self._datastore, self._project, self._mutations)
       logging.debug("Successfully wrote %d mutations.", len(self._mutations))
       self._mutations = []
+      self._mutations_size = 0
 
 
 class WriteToDatastore(_Mutate):
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
index 6adc08ad2848..424e714c5737 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+import math
 import unittest
 
 from mock import MagicMock, call, patch
@@ -191,6 +192,22 @@ def check_DatastoreWriteFn(self, num_entities):
       self.assertEqual((num_entities - 1) / _Mutate._WRITE_BATCH_SIZE + 1,
                        self._mock_datastore.commit.call_count)
 
+  def test_DatastoreWriteLargeEntities(self):
+    """100*100kB entities gets split over two Commit RPCs."""
+    with patch.object(helper, 'get_datastore',
+                      return_value=self._mock_datastore):
+      entities = [e.entity for e in fake_datastore.create_entities(100)]
+
+      datastore_write_fn = _Mutate.DatastoreWriteFn(self._PROJECT)
+      datastore_write_fn.start_bundle()
+      for entity in entities:
+        datastore_helper.add_properties(
+            entity, {'large': u'A' * 100000}, exclude_from_indexes=True)
+        datastore_write_fn.process(WriteToDatastore.to_upsert_mutation(entity))
+      datastore_write_fn.finish_bundle()
+
+      self.assertEqual(2, self._mock_datastore.commit.call_count)
+
   def verify_unique_keys(self, queries):
     """A helper function that verifies if all the queries have unique keys."""
     keys, _ = zip(*queries)

From 6ed15278650c326cf58ec1f2aa37855dbfc204eb Mon Sep 17 00:00:00 2001
From: Jason Kuster <jasonkuster@google.com>
Date: Mon, 22 May 2017 17:46:49 -0700
Subject: [PATCH 285/387] Re-rename fat jar so that install doesn't install the
 bundled jar as the default jar.

Signed-off-by: Jason Kuster <jasonkuster@google.com>
---
 .../src/main/resources/archetype-resources/pom.xml               | 1 +
 .../examples/src/main/resources/archetype-resources/pom.xml      | 1 +
 2 files changed, 2 insertions(+)

diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
index 47dfc91face3..af4fbd3832e3 100644
--- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml
@@ -114,6 +114,7 @@
               <goal>shade</goal>
             </goals>
             <configuration>
+              <finalName>${project.artifactId}-bundled-${project.version}</finalName>
               <filters>
                 <filter>
                   <artifact>*:*</artifact>
diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
index 025e3565a21c..b8b9c9f0fa49 100644
--- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
+++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -113,6 +113,7 @@
               <goal>shade</goal>
             </goals>
             <configuration>
+              <finalName>${project.artifactId}-bundled-${project.version}</finalName>
               <filters>
                 <filter>
                   <artifact>*:*</artifact>

From 94e619d4cbe90ee5d29af346c978d46ab567c951 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 20:27:52 -0700
Subject: [PATCH 286/387] Revert "Include Additional PTransform inputs in
 Transform Nodes"

This reverts commit 9336230d2a5c18bae89908bcd60db8ea96b7906d.

It appears to break the BigQueryTornadoesIT.
---
 .../apex/translation/TranslationContext.java  |   4 +-
 .../core/construction/TransformInputs.java    |  50 ------
 .../construction/TransformInputsTest.java     | 166 ------------------
 .../runners/direct/DirectGraphVisitor.java    |  15 +-
 .../runners/direct/ParDoEvaluatorFactory.java |   9 +-
 ...ttableProcessElementsEvaluatorFactory.java |   2 -
 .../direct/StatefulParDoEvaluatorFactory.java |   1 -
 .../beam/runners/direct/WatermarkManager.java |  17 +-
 .../runners/direct/ParDoEvaluatorTest.java    |   6 +-
 .../flink/FlinkBatchTranslationContext.java   |   3 +-
 .../FlinkStreamingTranslationContext.java     |   3 +-
 .../spark/translation/EvaluationContext.java  |   4 +-
 .../beam/sdk/runners/TransformHierarchy.java  |  28 +--
 13 files changed, 28 insertions(+), 280 deletions(-)
 delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
 delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index 94d13e177dec..aff3863624c4 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -34,7 +34,6 @@
 import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -94,8 +93,7 @@ public Map<TupleTag<?>, PValue> getInputs() {
   }
 
   public <InputT extends PValue> InputT getInput() {
-    return (InputT)
-        Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
+    return (InputT) Iterables.getOnlyElement(getCurrentTransform().getInputs().values());
   }
 
   public Map<TupleTag<?>, PValue> getOutputs() {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
deleted file mode 100644
index 2baf93a3c128..000000000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.collect.ImmutableList;
-import java.util.Collection;
-import java.util.Map;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
-
-/** Utilities for extracting subsets of inputs from an {@link AppliedPTransform}. */
-public class TransformInputs {
-  /**
-   * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link
-   * PTransform#getAdditionalInputs()}.
-   */
-  public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) {
-    ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder();
-    PTransform<?, ?> transform = application.getTransform();
-    for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) {
-      if (!transform.getAdditionalInputs().containsKey(input.getKey())) {
-        mainInputs.add(input.getValue());
-      }
-    }
-    checkArgument(
-        !mainInputs.build().isEmpty() || application.getInputs().isEmpty(),
-        "Expected at least one main input if any inputs exist");
-    return mainInputs.build();
-  }
-}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
deleted file mode 100644
index f5b2c11e7923..000000000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static org.junit.Assert.assertThat;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.GenerateSequence;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PDone;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.hamcrest.Matchers;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link TransformInputs}. */
-@RunWith(JUnit4.class)
-public class TransformInputsTest {
-  @Rule public TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false);
-  @Rule public ExpectedException thrown = ExpectedException.none();
-
-  @Test
-  public void nonAdditionalInputsWithNoInputSucceeds() {
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "input-free",
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(),
-            pipeline);
-
-    assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>empty());
-  }
-
-  @Test
-  public void nonAdditionalInputsWithOneMainInputSucceeds() {
-    PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L));
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "input-single",
-            Collections.<TupleTag<?>, PValue>singletonMap(new TupleTag<Long>() {}, input),
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(),
-            pipeline);
-
-    assertThat(
-        TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>containsInAnyOrder(input));
-  }
-
-  @Test
-  public void nonAdditionalInputsWithMultipleNonAdditionalInputsSucceeds() {
-    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
-    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
-    allInputs.put(new TupleTag<Integer>() {}, mainInts);
-    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
-    allInputs.put(new TupleTag<Void>() {}, voids);
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "additional-free",
-            allInputs,
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(),
-            pipeline);
-
-    assertThat(
-        TransformInputs.nonAdditionalInputs(transform),
-        Matchers.<PValue>containsInAnyOrder(voids, mainInts));
-  }
-
-  @Test
-  public void nonAdditionalInputsWithAdditionalInputsSucceeds() {
-    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
-    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
-    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
-
-    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
-    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
-    allInputs.put(new TupleTag<Integer>() {}, mainInts);
-    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
-    allInputs.put(
-        new TupleTag<Void>() {}, voids);
-    allInputs.putAll(additionalInputs);
-
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "additional",
-            allInputs,
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(additionalInputs),
-            pipeline);
-
-    assertThat(
-        TransformInputs.nonAdditionalInputs(transform),
-        Matchers.<PValue>containsInAnyOrder(mainInts, voids));
-  }
-
-  @Test
-  public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() {
-    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
-    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
-    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
-
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "additional-only",
-            additionalInputs,
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(additionalInputs),
-            pipeline);
-
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("at least one");
-    TransformInputs.nonAdditionalInputs(transform);
-  }
-
-  private static class TestTransform extends PTransform<PInput, POutput> {
-    private final Map<TupleTag<?>, PValue> additionalInputs;
-
-    private TestTransform() {
-      this(Collections.<TupleTag<?>, PValue>emptyMap());
-    }
-
-    private TestTransform(Map<TupleTag<?>, PValue> additionalInputs) {
-      this.additionalInputs = additionalInputs;
-    }
-
-    @Override
-    public POutput expand(PInput input) {
-      return PDone.in(input.getPipeline());
-    }
-
-    @Override
-    public Map<TupleTag<?>, PValue> getAdditionalInputs() {
-      return additionalInputs;
-    }
-  }
-}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index ed4282bbdb8b..01204e3049dd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -21,12 +21,10 @@
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -36,8 +34,6 @@
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Tracks the {@link AppliedPTransform AppliedPTransforms} that consume each {@link PValue} in the
@@ -45,7 +41,6 @@
  * input after the upstream transform has produced and committed output.
  */
 class DirectGraphVisitor extends PipelineVisitor.Defaults {
-  private static final Logger LOG = LoggerFactory.getLogger(DirectGraphVisitor.class);
 
   private Map<POutput, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
 
@@ -88,15 +83,7 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
     if (node.getInputs().isEmpty()) {
       rootTransforms.add(appliedTransform);
     } else {
-      Collection<PValue> mainInputs =
-          TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline()));
-      if (!mainInputs.containsAll(node.getInputs().values())) {
-        LOG.debug(
-            "Inputs reduced to {} from {} by removing additional inputs",
-            mainInputs,
-            node.getInputs().values());
-      }
-      for (PValue value : mainInputs) {
+      for (PValue value : node.getInputs().values()) {
         primitiveConsumers.put(value, appliedTransform);
       }
     }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index c52091e27cfe..74470bfb8b8d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -20,6 +20,7 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -78,7 +79,6 @@ public <T> TransformEvaluator<T> forApplication(
         (TransformEvaluator<T>)
             createEvaluator(
                 (AppliedPTransform) application,
-                (PCollection<InputT>) inputBundle.getPCollection(),
                 inputBundle.getKey(),
                 doFn,
                 transform.getSideInputs(),
@@ -102,7 +102,6 @@ public void cleanup() throws Exception {
   @SuppressWarnings({"unchecked", "rawtypes"})
   DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
-      PCollection<InputT> mainInput,
       StructuralKey<?> inputBundleKey,
       DoFn<InputT, OutputT> doFn,
       List<PCollectionView<?>> sideInputs,
@@ -121,7 +120,6 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
         createParDoEvaluator(
             application,
             inputBundleKey,
-            mainInput,
             sideInputs,
             mainOutputTag,
             additionalOutputTags,
@@ -134,7 +132,6 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
   ParDoEvaluator<InputT> createParDoEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
       StructuralKey<?> key,
-      PCollection<InputT> mainInput,
       List<PCollectionView<?>> sideInputs,
       TupleTag<OutputT> mainOutputTag,
       List<TupleTag<?>> additionalOutputTags,
@@ -147,7 +144,8 @@ ParDoEvaluator<InputT> createParDoEvaluator(
           evaluationContext,
           stepContext,
           application,
-          mainInput.getWindowingStrategy(),
+          ((PCollection<InputT>) Iterables.getOnlyElement(application.getInputs().values()))
+              .getWindowingStrategy(),
           fn,
           key,
           sideInputs,
@@ -175,4 +173,5 @@ static Map<TupleTag<?>, PCollection<?>> pcollections(Map<TupleTag<?>, PValue> ou
     }
     return pcs;
   }
+
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index 4e7f4db65478..dc85d87bc93f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -116,8 +116,6 @@ public void cleanup() throws Exception {
             delegateFactory.createParDoEvaluator(
                 application,
                 inputBundle.getKey(),
-                (PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>)
-                    inputBundle.getPCollection(),
                 transform.getSideInputs(),
                 transform.getMainOutputTag(),
                 transform.getAdditionalOutputTags().getAll(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index e22edd187c53..985c3be4e9e9 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -117,7 +117,6 @@ private TransformEvaluator<KeyedWorkItem<K, KV<K, InputT>>> createEvaluator(
     DoFnLifecycleManagerRemovingTransformEvaluator<KV<K, InputT>> delegateEvaluator =
         delegateFactory.createEvaluator(
             (AppliedPTransform) application,
-            (PCollection) inputBundle.getPCollection(),
             inputBundle.getKey(),
             doFn,
             application.getTransform().getUnderlyingParDo().getSideInputs(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index b15b52e314de..4f1b8319dc2d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -823,11 +823,10 @@ private Collection<Watermark> getInputProcessingWatermarks(AppliedPTransform<?,
       inputWmsBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
-        Watermark producerOutputWatermark =
-            getTransformWatermark(graph.getProducer(pvalue)).synchronizedProcessingOutputWatermark;
-        inputWmsBuilder.add(producerOutputWatermark);
-      }
+      Watermark producerOutputWatermark =
+          getTransformWatermark(graph.getProducer(pvalue))
+              .synchronizedProcessingOutputWatermark;
+      inputWmsBuilder.add(producerOutputWatermark);
     }
     return inputWmsBuilder.build();
   }
@@ -839,11 +838,9 @@ private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform)
       inputWatermarksBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
-        Watermark producerOutputWatermark =
-            getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
-        inputWatermarksBuilder.add(producerOutputWatermark);
-      }
+      Watermark producerOutputWatermark =
+          getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
+      inputWatermarksBuilder.add(producerOutputWatermark);
     }
     List<Watermark> inputCollectionWatermarks = inputWatermarksBuilder.build();
     return inputCollectionWatermarks;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 3b2a22ee26f9..286e44d1be04 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -98,7 +98,7 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
     when(evaluationContext.createBundle(output)).thenReturn(outputBundle);
 
     ParDoEvaluator<Integer> evaluator =
-        createEvaluator(singletonView, fn, inputPc, output);
+        createEvaluator(singletonView, fn, output);
 
     IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
     WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
@@ -132,7 +132,6 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
   private ParDoEvaluator<Integer> createEvaluator(
       PCollectionView<Integer> singletonView,
       RecorderFn fn,
-      PCollection<Integer> input,
       PCollection<Integer> output) {
     when(
             evaluationContext.createSideInputReader(
@@ -157,7 +156,8 @@ private ParDoEvaluator<Integer> createEvaluator(
         evaluationContext,
         stepContext,
         transform,
-        input.getWindowingStrategy(),
+        ((PCollection<?>) Iterables.getOnlyElement(transform.getInputs().values()))
+            .getWindowingStrategy(),
         fn,
         null /* key */,
         ImmutableList.<PCollectionView<?>>of(singletonView),
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index 6e7019848b19..0439119dfc40 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -20,7 +20,6 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -144,7 +143,7 @@ Map<TupleTag<?>, PValue> getInputs(PTransform<?, ?> transform) {
 
   @SuppressWarnings("unchecked")
   <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
+    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
   }
 
   Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
index 74a5fb971144..ea5f6b3162af 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
@@ -22,7 +22,6 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -114,7 +113,7 @@ public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collecti
 
   @SuppressWarnings("unchecked")
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
+    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
   }
 
   public <T extends PInput> Map<TupleTag<?>, PValue> getInputs(PTransform<T, ?> transform) {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 0c6c4d1cb660..8102926f6daa 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -26,7 +26,6 @@
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.sdk.Pipeline;
@@ -104,8 +103,7 @@ public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
 
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
     @SuppressWarnings("unchecked")
-    T input =
-        (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
+    T input = (T) Iterables.getOnlyElement(getInputs(transform).values());
     return input;
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 9d73b4576788..2f0e8efd7de8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
@@ -67,7 +68,7 @@ public TransformHierarchy() {
     producers = new HashMap<>();
     producerInput = new HashMap<>();
     unexpandedInputs = new HashMap<>();
-    root = new Node();
+    root = new Node(null, null, "", null);
     current = root;
   }
 
@@ -251,37 +252,26 @@ public class Node {
     @VisibleForTesting
     boolean finishedSpecifying = false;
 
-    /**
-     * Creates the root-level node. The root level node has a null enclosing node, a null transform,
-     * an empty map of inputs, and a name equal to the empty string.
-     */
-    private Node() {
-      this.enclosingNode = null;
-      this.transform = null;
-      this.fullName = "";
-      this.inputs = Collections.emptyMap();
-    }
-
     /**
      * Creates a new Node with the given parent and transform.
      *
+     * <p>EnclosingNode and transform may both be null for a root-level node, which holds all other
+     * nodes.
+     *
      * @param enclosingNode the composite node containing this node
      * @param transform the PTransform tracked by this node
      * @param fullName the fully qualified name of the transform
      * @param input the unexpanded input to the transform
      */
     private Node(
-        Node enclosingNode,
-        PTransform<?, ?> transform,
+        @Nullable Node enclosingNode,
+        @Nullable PTransform<?, ?> transform,
         String fullName,
-        PInput input) {
+        @Nullable PInput input) {
       this.enclosingNode = enclosingNode;
       this.transform = transform;
       this.fullName = fullName;
-      ImmutableMap.Builder<TupleTag<?>, PValue> inputs = ImmutableMap.builder();
-      inputs.putAll(input.expand());
-      inputs.putAll(transform.getAdditionalInputs());
-      this.inputs = inputs.build();
+      this.inputs = input == null ? Collections.<TupleTag<?>, PValue>emptyMap() : input.expand();
     }
 
     /**

From 3a62b4f7b20bda2b3c4ca648f90988d387cfe20d Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 18 May 2017 17:22:25 -0700
Subject: [PATCH 287/387] Automatically convert examples to use with syntax.

---
 .../examples/complete/autocomplete.py         |  19 +-
 .../examples/complete/autocomplete_test.py    |  31 +-
 .../examples/complete/estimate_pi.py          |  11 +-
 .../examples/complete/estimate_pi_test.py     |  12 +-
 .../complete/game/hourly_team_score.py        |  19 +-
 .../examples/complete/game/user_score.py      |  15 +-
 .../complete/juliaset/juliaset/juliaset.py    |  44 +-
 .../apache_beam/examples/complete/tfidf.py    |  21 +-
 .../examples/complete/tfidf_test.py           |  28 +-
 .../complete/top_wikipedia_sessions.py        |  12 +-
 .../complete/top_wikipedia_sessions_test.py   |   9 +-
 .../examples/cookbook/bigquery_schema.py      | 159 +++---
 .../examples/cookbook/bigquery_side_input.py  |  51 +-
 .../cookbook/bigquery_side_input_test.py      |  39 +-
 .../examples/cookbook/bigquery_tornadoes.py   |  33 +-
 .../cookbook/bigquery_tornadoes_test.py       |  19 +-
 .../apache_beam/examples/cookbook/coders.py   |  16 +-
 .../examples/cookbook/coders_test.py          |  14 +-
 .../examples/cookbook/custom_ptransform.py    |  27 +-
 .../cookbook/custom_ptransform_test.py        |  11 +-
 .../examples/cookbook/datastore_wordcount.py  |  20 +-
 .../apache_beam/examples/cookbook/filters.py  |  21 +-
 .../examples/cookbook/group_with_coder.py     |  43 +-
 .../cookbook/group_with_coder_test.py         |   4 +-
 .../examples/cookbook/mergecontacts.py        | 115 ++--
 .../examples/cookbook/mergecontacts_test.py   |   3 +-
 .../cookbook/multiple_output_pardo.py         |  72 ++-
 .../cookbook/multiple_output_pardo_test.py    |   2 +-
 .../apache_beam/examples/snippets/snippets.py | 494 +++++++++---------
 .../examples/snippets/snippets_test.py        | 326 ++++++------
 .../apache_beam/examples/streaming_wordcap.py |  24 +-
 .../examples/streaming_wordcount.py           |  44 +-
 sdks/python/apache_beam/examples/wordcount.py |   1 -
 .../examples/wordcount_debugging.py           |  55 +-
 .../apache_beam/examples/wordcount_minimal.py |  33 +-
 .../apache_beam/io/filebasedsink_test.py      |  16 +-
 sdks/python/apache_beam/pipeline.py           |  19 +-
 .../apache_beam/transforms/combiners_test.py  |  58 +-
 .../apache_beam/transforms/window_test.py     | 147 +++---
 .../transforms/write_ptransform_test.py       |   7 +-
 .../typehints/typed_pipeline_test.py          |  22 +-
 41 files changed, 1005 insertions(+), 1111 deletions(-)

diff --git a/sdks/python/apache_beam/examples/complete/autocomplete.py b/sdks/python/apache_beam/examples/complete/autocomplete.py
index f0acc3fc562d..ab3397cfe335 100644
--- a/sdks/python/apache_beam/examples/complete/autocomplete.py
+++ b/sdks/python/apache_beam/examples/complete/autocomplete.py
@@ -44,16 +44,15 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  (p  # pylint: disable=expression-not-assigned
-   | 'read' >> ReadFromText(known_args.input)
-   | 'split' >> beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
-   | 'TopPerPrefix' >> TopPerPrefix(5)
-   | 'format' >> beam.Map(
-       lambda (prefix, candidates): '%s: %s' % (prefix, candidates))
-   | 'write' >> WriteToText(known_args.output))
-  p.run()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    (p  # pylint: disable=expression-not-assigned
+     | 'read' >> ReadFromText(known_args.input)
+     | 'split' >> beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+     | 'TopPerPrefix' >> TopPerPrefix(5)
+     | 'format' >> beam.Map(
+         lambda (prefix, candidates): '%s: %s' % (prefix, candidates))
+     | 'write' >> WriteToText(known_args.output))
 
 
 class TopPerPrefix(beam.PTransform):
diff --git a/sdks/python/apache_beam/examples/complete/autocomplete_test.py b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
index 378d222bfa8d..e2c84d68d3d4 100644
--- a/sdks/python/apache_beam/examples/complete/autocomplete_test.py
+++ b/sdks/python/apache_beam/examples/complete/autocomplete_test.py
@@ -31,22 +31,21 @@ class AutocompleteTest(unittest.TestCase):
   WORDS = ['this', 'this', 'that', 'to', 'to', 'to']
 
   def test_top_prefixes(self):
-    p = TestPipeline()
-    words = p | beam.Create(self.WORDS)
-    result = words | autocomplete.TopPerPrefix(5)
-    # values must be hashable for now
-    result = result | beam.Map(lambda (k, vs): (k, tuple(vs)))
-    assert_that(result, equal_to(
-        [
-            ('t', ((3, 'to'), (2, 'this'), (1, 'that'))),
-            ('to', ((3, 'to'), )),
-            ('th', ((2, 'this'), (1, 'that'))),
-            ('thi', ((2, 'this'), )),
-            ('this', ((2, 'this'), )),
-            ('tha', ((1, 'that'), )),
-            ('that', ((1, 'that'), )),
-        ]))
-    p.run()
+    with TestPipeline() as p:
+      words = p | beam.Create(self.WORDS)
+      result = words | autocomplete.TopPerPrefix(5)
+      # values must be hashable for now
+      result = result | beam.Map(lambda (k, vs): (k, tuple(vs)))
+      assert_that(result, equal_to(
+          [
+              ('t', ((3, 'to'), (2, 'this'), (1, 'that'))),
+              ('to', ((3, 'to'), )),
+              ('th', ((2, 'this'), (1, 'that'))),
+              ('thi', ((2, 'this'), )),
+              ('this', ((2, 'this'), )),
+              ('tha', ((1, 'that'), )),
+              ('that', ((1, 'that'), )),
+          ]))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py
index c709713bd97e..7e3c4cd35a27 100644
--- a/sdks/python/apache_beam/examples/complete/estimate_pi.py
+++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py
@@ -113,14 +113,11 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
+  with beam.Pipeline(options=pipeline_options) as p:
 
-  (p  # pylint: disable=expression-not-assigned
-   | EstimatePiTransform()
-   | WriteToText(known_args.output, coder=JsonCoder()))
-
-  # Actually run the pipeline (all operations above are deferred).
-  p.run()
+    (p  # pylint: disable=expression-not-assigned
+     | EstimatePiTransform()
+     | WriteToText(known_args.output, coder=JsonCoder()))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
index fd5130966887..f1cbb0a24d56 100644
--- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
+++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py
@@ -38,13 +38,13 @@ def _in_between(actual):
 class EstimatePiTest(unittest.TestCase):
 
   def test_basics(self):
-    p = TestPipeline()
-    result = p | 'Estimate' >> estimate_pi.EstimatePiTransform(5000)
+    with TestPipeline() as p:
+      result = p | 'Estimate' >> estimate_pi.EstimatePiTransform(5000)
 
-    # Note: Probabilistically speaking this test can fail with a probability
-    # that is very small (VERY) given that we run at least 500 thousand trials.
-    assert_that(result, in_between(3.125, 3.155))
-    p.run()
+      # Note: Probabilistically speaking this test can fail with a probability
+      # that is very small (VERY) given that we run at least 500 thousand
+      # trials.
+      assert_that(result, in_between(3.125, 3.155))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
index e9d71881f8c9..9f398d9995f2 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
@@ -276,18 +276,15 @@ def run(argv=None):
   known_args, pipeline_args = parser.parse_known_args(argv)
 
   pipeline_options = PipelineOptions(pipeline_args)
-  p = beam.Pipeline(options=pipeline_options)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-
-  (p  # pylint: disable=expression-not-assigned
-   | ReadFromText(known_args.input)
-   | HourlyTeamScore(
-       known_args.start_min, known_args.stop_min, known_args.window_duration)
-   | WriteWindowedToBigQuery(
-       known_args.table_name, known_args.dataset, configure_bigquery_write()))
-
-  result = p.run()
-  result.wait_until_finish()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    (p  # pylint: disable=expression-not-assigned
+     | ReadFromText(known_args.input)
+     | HourlyTeamScore(
+         known_args.start_min, known_args.stop_min, known_args.window_duration)
+     | WriteWindowedToBigQuery(
+         known_args.table_name, known_args.dataset, configure_bigquery_write()))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/game/user_score.py b/sdks/python/apache_beam/examples/complete/game/user_score.py
index 389d2c6a8433..c9f273814a85 100644
--- a/sdks/python/apache_beam/examples/complete/game/user_score.py
+++ b/sdks/python/apache_beam/examples/complete/game/user_score.py
@@ -201,16 +201,13 @@ def run(argv=None):
   known_args, pipeline_args = parser.parse_known_args(argv)
 
   pipeline_options = PipelineOptions(pipeline_args)
-  p = beam.Pipeline(options=pipeline_options)
+  with beam.Pipeline(options=pipeline_options) as p:
 
-  (p  # pylint: disable=expression-not-assigned
-   | ReadFromText(known_args.input) # Read events from a file and parse them.
-   | UserScore()
-   | WriteToBigQuery(
-       known_args.table_name, known_args.dataset, configure_bigquery_write()))
-
-  result = p.run()
-  result.wait_until_finish()
+    (p  # pylint: disable=expression-not-assigned
+     | ReadFromText(known_args.input) # Read events from a file and parse them.
+     | UserScore()
+     | WriteToBigQuery(
+         known_args.table_name, known_args.dataset, configure_bigquery_write()))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py
index 5ff2b785108b..61e3fd1a8d0c 100644
--- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py
+++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset.py
@@ -99,26 +99,24 @@ def run(argv=None):  # pylint: disable=missing-docstring
                       help='Output file to write the resulting image to.')
   known_args, pipeline_args = parser.parse_known_args(argv)
 
-  p = beam.Pipeline(argv=pipeline_args)
-  n = int(known_args.grid_size)
-
-  coordinates = generate_julia_set_colors(p, complex(-.62772, .42193), n, 100)
-
-  # Group each coordinate triplet by its x value, then write the coordinates to
-  # the output file with an x-coordinate grouping per line.
-  # pylint: disable=expression-not-assigned
-  (coordinates
-   | 'x coord key' >> beam.Map(lambda (x, y, i): (x, (x, y, i)))
-   | 'x coord' >> beam.GroupByKey()
-   | 'format' >> beam.Map(
-       lambda (k, coords): ' '.join('(%s, %s, %s)' % coord for coord in coords))
-   | WriteToText(known_args.coordinate_output))
-  # pylint: enable=expression-not-assigned
-  return p.run().wait_until_finish()
-
-  # Optionally render the image and save it to a file.
-  # TODO(silviuc): Add this functionality.
-  # if p.options.image_output is not None:
-  #  julia_set_image = generate_julia_set_visualization(
-  #      file_with_coordinates, n, 100)
-  #  save_julia_set_visualization(p.options.image_output, julia_set_image)
+  with beam.Pipeline(argv=pipeline_args) as p:
+    n = int(known_args.grid_size)
+
+    coordinates = generate_julia_set_colors(p, complex(-.62772, .42193), n, 100)
+
+    # Group each coordinate triplet by its x value, then write the coordinates
+    # to the output file with an x-coordinate grouping per line.
+    # pylint: disable=expression-not-assigned
+    (coordinates
+     | 'x coord key' >> beam.Map(lambda (x, y, i): (x, (x, y, i)))
+     | 'x coord' >> beam.GroupByKey()
+     | 'format' >> beam.Map(
+         lambda (k, coords): ' '.join('(%s, %s, %s)' % c for c in coords))
+     | WriteToText(known_args.coordinate_output))
+
+    # Optionally render the image and save it to a file.
+    # TODO(silviuc): Add this functionality.
+    # if p.options.image_output is not None:
+    #  julia_set_image = generate_julia_set_visualization(
+    #      file_with_coordinates, n, 100)
+    #  save_julia_set_visualization(p.options.image_output, julia_set_image)
diff --git a/sdks/python/apache_beam/examples/complete/tfidf.py b/sdks/python/apache_beam/examples/complete/tfidf.py
index a98d90640d52..a88ff827766c 100644
--- a/sdks/python/apache_beam/examples/complete/tfidf.py
+++ b/sdks/python/apache_beam/examples/complete/tfidf.py
@@ -191,17 +191,16 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  # Read documents specified by the uris command line option.
-  pcoll = read_documents(p, glob.glob(known_args.uris))
-  # Compute TF-IDF information for each word.
-  output = pcoll | TfIdf()
-  # Write the output using a "Write" transform that has side effects.
-  # pylint: disable=expression-not-assigned
-  output | 'write' >> WriteToText(known_args.output)
-  # Execute the pipeline and wait until it is completed.
-  p.run().wait_until_finish()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    # Read documents specified by the uris command line option.
+    pcoll = read_documents(p, glob.glob(known_args.uris))
+    # Compute TF-IDF information for each word.
+    output = pcoll | TfIdf()
+    # Write the output using a "Write" transform that has side effects.
+    # pylint: disable=expression-not-assigned
+    output | 'write' >> WriteToText(known_args.output)
+    # Execute the pipeline and wait until it is completed.
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/tfidf_test.py b/sdks/python/apache_beam/examples/complete/tfidf_test.py
index f177dfc9e166..322426fd2b3d 100644
--- a/sdks/python/apache_beam/examples/complete/tfidf_test.py
+++ b/sdks/python/apache_beam/examples/complete/tfidf_test.py
@@ -50,20 +50,20 @@ def create_file(self, path, contents):
       f.write(contents)
 
   def test_tfidf_transform(self):
-    p = TestPipeline()
-    uri_to_line = p | 'create sample' >> beam.Create(
-        [('1.txt', 'abc def ghi'),
-         ('2.txt', 'abc def'),
-         ('3.txt', 'abc')])
-    result = (
-        uri_to_line
-        | tfidf.TfIdf()
-        | beam.Map(lambda (word, (uri, tfidf)): (word, uri, tfidf)))
-    assert_that(result, equal_to(EXPECTED_RESULTS))
-    # Run the pipeline. Note that the assert_that above adds to the pipeline
-    # a check that the result PCollection contains expected values. To actually
-    # trigger the check the pipeline must be run.
-    p.run()
+    with TestPipeline() as p:
+      uri_to_line = p | 'create sample' >> beam.Create(
+          [('1.txt', 'abc def ghi'),
+           ('2.txt', 'abc def'),
+           ('3.txt', 'abc')])
+      result = (
+          uri_to_line
+          | tfidf.TfIdf()
+          | beam.Map(lambda (word, (uri, tfidf)): (word, uri, tfidf)))
+      assert_that(result, equal_to(EXPECTED_RESULTS))
+      # Run the pipeline. Note that the assert_that above adds to the pipeline
+      # a check that the result PCollection contains expected values.
+      # To actually trigger the check the pipeline must be run (e.g. by
+      # exiting the with context).
 
   def test_basics(self):
     # Setup the files with expected content.
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
index aa48e4eb920e..9a9ad7865c2c 100644
--- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
+++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
@@ -159,14 +159,12 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
+  with beam.Pipeline(options=pipeline_options) as p:
 
-  (p  # pylint: disable=expression-not-assigned
-   | ReadFromText(known_args.input)
-   | ComputeTopSessions(known_args.sampling_threshold)
-   | WriteToText(known_args.output))
-
-  p.run()
+    (p  # pylint: disable=expression-not-assigned
+     | ReadFromText(known_args.input)
+     | ComputeTopSessions(known_args.sampling_threshold)
+     | WriteToText(known_args.output))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
index 5fb6276cb8d5..ced8a44af13c 100644
--- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
+++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py
@@ -52,12 +52,11 @@ class ComputeTopSessionsTest(unittest.TestCase):
   ]
 
   def test_compute_top_sessions(self):
-    p = TestPipeline()
-    edits = p | beam.Create(self.EDITS)
-    result = edits | top_wikipedia_sessions.ComputeTopSessions(1.0)
+    with TestPipeline() as p:
+      edits = p | beam.Create(self.EDITS)
+      result = edits | top_wikipedia_sessions.ComputeTopSessions(1.0)
 
-    assert_that(result, equal_to(self.EXPECTED))
-    p.run()
+      assert_that(result, equal_to(self.EXPECTED))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py b/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py
index 400189e64424..3a8af67d1391 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_schema.py
@@ -42,86 +42,85 @@ def run(argv=None):
        'or DATASET.TABLE.'))
   known_args, pipeline_args = parser.parse_known_args(argv)
 
-  p = beam.Pipeline(argv=pipeline_args)
-
-  from apache_beam.io.gcp.internal.clients import bigquery  # pylint: disable=wrong-import-order, wrong-import-position
-
-  table_schema = bigquery.TableSchema()
-
-  # Fields that use standard types.
-  kind_schema = bigquery.TableFieldSchema()
-  kind_schema.name = 'kind'
-  kind_schema.type = 'string'
-  kind_schema.mode = 'nullable'
-  table_schema.fields.append(kind_schema)
-
-  full_name_schema = bigquery.TableFieldSchema()
-  full_name_schema.name = 'fullName'
-  full_name_schema.type = 'string'
-  full_name_schema.mode = 'required'
-  table_schema.fields.append(full_name_schema)
-
-  age_schema = bigquery.TableFieldSchema()
-  age_schema.name = 'age'
-  age_schema.type = 'integer'
-  age_schema.mode = 'nullable'
-  table_schema.fields.append(age_schema)
-
-  gender_schema = bigquery.TableFieldSchema()
-  gender_schema.name = 'gender'
-  gender_schema.type = 'string'
-  gender_schema.mode = 'nullable'
-  table_schema.fields.append(gender_schema)
-
-  # A nested field
-  phone_number_schema = bigquery.TableFieldSchema()
-  phone_number_schema.name = 'phoneNumber'
-  phone_number_schema.type = 'record'
-  phone_number_schema.mode = 'nullable'
-
-  area_code = bigquery.TableFieldSchema()
-  area_code.name = 'areaCode'
-  area_code.type = 'integer'
-  area_code.mode = 'nullable'
-  phone_number_schema.fields.append(area_code)
-
-  number = bigquery.TableFieldSchema()
-  number.name = 'number'
-  number.type = 'integer'
-  number.mode = 'nullable'
-  phone_number_schema.fields.append(number)
-  table_schema.fields.append(phone_number_schema)
-
-  # A repeated field.
-  children_schema = bigquery.TableFieldSchema()
-  children_schema.name = 'children'
-  children_schema.type = 'string'
-  children_schema.mode = 'repeated'
-  table_schema.fields.append(children_schema)
-
-  def create_random_record(record_id):
-    return {'kind': 'kind' + record_id, 'fullName': 'fullName'+record_id,
-            'age': int(record_id) * 10, 'gender': 'male',
-            'phoneNumber': {
-                'areaCode': int(record_id) * 100,
-                'number': int(record_id) * 100000},
-            'children': ['child' + record_id + '1',
-                         'child' + record_id + '2',
-                         'child' + record_id + '3']
-           }
-
-  # pylint: disable=expression-not-assigned
-  record_ids = p | 'CreateIDs' >> beam.Create(['1', '2', '3', '4', '5'])
-  records = record_ids | 'CreateRecords' >> beam.Map(create_random_record)
-  records | 'write' >> beam.io.Write(
-      beam.io.BigQuerySink(
-          known_args.output,
-          schema=table_schema,
-          create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
-          write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
-
-  # Run the pipeline (all operations are deferred until run() is called).
-  p.run()
+  with beam.Pipeline(argv=pipeline_args) as p:
+
+    from apache_beam.io.gcp.internal.clients import bigquery  # pylint: disable=wrong-import-order, wrong-import-position
+
+    table_schema = bigquery.TableSchema()
+
+    # Fields that use standard types.
+    kind_schema = bigquery.TableFieldSchema()
+    kind_schema.name = 'kind'
+    kind_schema.type = 'string'
+    kind_schema.mode = 'nullable'
+    table_schema.fields.append(kind_schema)
+
+    full_name_schema = bigquery.TableFieldSchema()
+    full_name_schema.name = 'fullName'
+    full_name_schema.type = 'string'
+    full_name_schema.mode = 'required'
+    table_schema.fields.append(full_name_schema)
+
+    age_schema = bigquery.TableFieldSchema()
+    age_schema.name = 'age'
+    age_schema.type = 'integer'
+    age_schema.mode = 'nullable'
+    table_schema.fields.append(age_schema)
+
+    gender_schema = bigquery.TableFieldSchema()
+    gender_schema.name = 'gender'
+    gender_schema.type = 'string'
+    gender_schema.mode = 'nullable'
+    table_schema.fields.append(gender_schema)
+
+    # A nested field
+    phone_number_schema = bigquery.TableFieldSchema()
+    phone_number_schema.name = 'phoneNumber'
+    phone_number_schema.type = 'record'
+    phone_number_schema.mode = 'nullable'
+
+    area_code = bigquery.TableFieldSchema()
+    area_code.name = 'areaCode'
+    area_code.type = 'integer'
+    area_code.mode = 'nullable'
+    phone_number_schema.fields.append(area_code)
+
+    number = bigquery.TableFieldSchema()
+    number.name = 'number'
+    number.type = 'integer'
+    number.mode = 'nullable'
+    phone_number_schema.fields.append(number)
+    table_schema.fields.append(phone_number_schema)
+
+    # A repeated field.
+    children_schema = bigquery.TableFieldSchema()
+    children_schema.name = 'children'
+    children_schema.type = 'string'
+    children_schema.mode = 'repeated'
+    table_schema.fields.append(children_schema)
+
+    def create_random_record(record_id):
+      return {'kind': 'kind' + record_id, 'fullName': 'fullName'+record_id,
+              'age': int(record_id) * 10, 'gender': 'male',
+              'phoneNumber': {
+                  'areaCode': int(record_id) * 100,
+                  'number': int(record_id) * 100000},
+              'children': ['child' + record_id + '1',
+                           'child' + record_id + '2',
+                           'child' + record_id + '3']
+             }
+
+    # pylint: disable=expression-not-assigned
+    record_ids = p | 'CreateIDs' >> beam.Create(['1', '2', '3', '4', '5'])
+    records = record_ids | 'CreateRecords' >> beam.Map(create_random_record)
+    records | 'write' >> beam.io.Write(
+        beam.io.BigQuerySink(
+            known_args.output,
+            schema=table_schema,
+            create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+            write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
+
+    # Run the pipeline (all operations are deferred until run() is called).
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
index 6b2881828a0d..9911a6716bd8 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py
@@ -88,32 +88,31 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  group_ids = []
-  for i in xrange(0, int(known_args.num_groups)):
-    group_ids.append('id' + str(i))
-
-  query_corpus = 'select UNIQUE(corpus) from publicdata:samples.shakespeare'
-  query_word = 'select UNIQUE(word) from publicdata:samples.shakespeare'
-  ignore_corpus = known_args.ignore_corpus
-  ignore_word = known_args.ignore_word
-
-  pcoll_corpus = p | 'read corpus' >> beam.io.Read(
-      beam.io.BigQuerySource(query=query_corpus))
-  pcoll_word = p | 'read_words' >> beam.io.Read(
-      beam.io.BigQuerySource(query=query_word))
-  pcoll_ignore_corpus = p | 'create_ignore_corpus' >> beam.Create(
-      [ignore_corpus])
-  pcoll_ignore_word = p | 'create_ignore_word' >> beam.Create([ignore_word])
-  pcoll_group_ids = p | 'create groups' >> beam.Create(group_ids)
-
-  pcoll_groups = create_groups(pcoll_group_ids, pcoll_corpus, pcoll_word,
-                               pcoll_ignore_corpus, pcoll_ignore_word)
-
-  # pylint:disable=expression-not-assigned
-  pcoll_groups | WriteToText(known_args.output)
-  p.run()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    group_ids = []
+    for i in xrange(0, int(known_args.num_groups)):
+      group_ids.append('id' + str(i))
+
+    query_corpus = 'select UNIQUE(corpus) from publicdata:samples.shakespeare'
+    query_word = 'select UNIQUE(word) from publicdata:samples.shakespeare'
+    ignore_corpus = known_args.ignore_corpus
+    ignore_word = known_args.ignore_word
+
+    pcoll_corpus = p | 'read corpus' >> beam.io.Read(
+        beam.io.BigQuerySource(query=query_corpus))
+    pcoll_word = p | 'read_words' >> beam.io.Read(
+        beam.io.BigQuerySource(query=query_word))
+    pcoll_ignore_corpus = p | 'create_ignore_corpus' >> beam.Create(
+        [ignore_corpus])
+    pcoll_ignore_word = p | 'create_ignore_word' >> beam.Create([ignore_word])
+    pcoll_group_ids = p | 'create groups' >> beam.Create(group_ids)
+
+    pcoll_groups = create_groups(pcoll_group_ids, pcoll_corpus, pcoll_word,
+                                 pcoll_ignore_corpus, pcoll_ignore_word)
+
+    # pylint:disable=expression-not-assigned
+    pcoll_groups | WriteToText(known_args.output)
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
index b11dc47c3363..964b35b3f08f 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py
@@ -30,25 +30,26 @@
 class BigQuerySideInputTest(unittest.TestCase):
 
   def test_create_groups(self):
-    p = TestPipeline()
-
-    group_ids_pcoll = p | 'CreateGroupIds' >> beam.Create(['A', 'B', 'C'])
-    corpus_pcoll = p | 'CreateCorpus' >> beam.Create(
-        [{'f': 'corpus1'}, {'f': 'corpus2'}, {'f': 'corpus3'}])
-    words_pcoll = p | 'CreateWords' >> beam.Create(
-        [{'f': 'word1'}, {'f': 'word2'}, {'f': 'word3'}])
-    ignore_corpus_pcoll = p | 'CreateIgnoreCorpus' >> beam.Create(['corpus1'])
-    ignore_word_pcoll = p | 'CreateIgnoreWord' >> beam.Create(['word1'])
-
-    groups = bigquery_side_input.create_groups(group_ids_pcoll, corpus_pcoll,
-                                               words_pcoll, ignore_corpus_pcoll,
-                                               ignore_word_pcoll)
-
-    assert_that(groups, equal_to(
-        [('A', 'corpus2', 'word2'),
-         ('B', 'corpus2', 'word2'),
-         ('C', 'corpus2', 'word2')]))
-    p.run()
+    with TestPipeline() as p:
+
+      group_ids_pcoll = p | 'CreateGroupIds' >> beam.Create(['A', 'B', 'C'])
+      corpus_pcoll = p | 'CreateCorpus' >> beam.Create(
+          [{'f': 'corpus1'}, {'f': 'corpus2'}, {'f': 'corpus3'}])
+      words_pcoll = p | 'CreateWords' >> beam.Create(
+          [{'f': 'word1'}, {'f': 'word2'}, {'f': 'word3'}])
+      ignore_corpus_pcoll = p | 'CreateIgnoreCorpus' >> beam.Create(['corpus1'])
+      ignore_word_pcoll = p | 'CreateIgnoreWord' >> beam.Create(['word1'])
+
+      groups = bigquery_side_input.create_groups(group_ids_pcoll,
+                                                 corpus_pcoll,
+                                                 words_pcoll,
+                                                 ignore_corpus_pcoll,
+                                                 ignore_word_pcoll)
+
+      assert_that(groups, equal_to(
+          [('A', 'corpus2', 'word2'),
+           ('B', 'corpus2', 'word2'),
+           ('C', 'corpus2', 'word2')]))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py
index ed0c79a71a15..d3b216e6dd3a 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py
@@ -75,23 +75,22 @@ def run(argv=None):
        'or DATASET.TABLE.'))
   known_args, pipeline_args = parser.parse_known_args(argv)
 
-  p = beam.Pipeline(argv=pipeline_args)
-
-  # Read the table rows into a PCollection.
-  rows = p | 'read' >> beam.io.Read(beam.io.BigQuerySource(known_args.input))
-  counts = count_tornadoes(rows)
-
-  # Write the output using a "Write" transform that has side effects.
-  # pylint: disable=expression-not-assigned
-  counts | 'write' >> beam.io.Write(
-      beam.io.BigQuerySink(
-          known_args.output,
-          schema='month:INTEGER, tornado_count:INTEGER',
-          create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
-          write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
-
-  # Run the pipeline (all operations are deferred until run() is called).
-  p.run().wait_until_finish()
+  with beam.Pipeline(argv=pipeline_args) as p:
+
+    # Read the table rows into a PCollection.
+    rows = p | 'read' >> beam.io.Read(beam.io.BigQuerySource(known_args.input))
+    counts = count_tornadoes(rows)
+
+    # Write the output using a "Write" transform that has side effects.
+    # pylint: disable=expression-not-assigned
+    counts | 'write' >> beam.io.Write(
+        beam.io.BigQuerySink(
+            known_args.output,
+            schema='month:INTEGER, tornado_count:INTEGER',
+            create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+            write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
+
+    # Run the pipeline (all operations are deferred until run() is called).
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
index c926df8ec15c..45dcabaf853c 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py
@@ -30,16 +30,15 @@
 class BigQueryTornadoesTest(unittest.TestCase):
 
   def test_basics(self):
-    p = TestPipeline()
-    rows = (p | 'create' >> beam.Create([
-        {'month': 1, 'day': 1, 'tornado': False},
-        {'month': 1, 'day': 2, 'tornado': True},
-        {'month': 1, 'day': 3, 'tornado': True},
-        {'month': 2, 'day': 1, 'tornado': True}]))
-    results = bigquery_tornadoes.count_tornadoes(rows)
-    assert_that(results, equal_to([{'month': 1, 'tornado_count': 2},
-                                   {'month': 2, 'tornado_count': 1}]))
-    p.run().wait_until_finish()
+    with TestPipeline() as p:
+      rows = (p | 'create' >> beam.Create([
+          {'month': 1, 'day': 1, 'tornado': False},
+          {'month': 1, 'day': 2, 'tornado': True},
+          {'month': 1, 'day': 3, 'tornado': True},
+          {'month': 2, 'day': 1, 'tornado': True}]))
+      results = bigquery_tornadoes.count_tornadoes(rows)
+      assert_that(results, equal_to([{'month': 1, 'tornado_count': 2},
+                                     {'month': 2, 'tornado_count': 1}]))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/coders.py b/sdks/python/apache_beam/examples/cookbook/coders.py
index aeeb3c9128a5..f97b0f2d9c9a 100644
--- a/sdks/python/apache_beam/examples/cookbook/coders.py
+++ b/sdks/python/apache_beam/examples/cookbook/coders.py
@@ -85,15 +85,13 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  p = beam.Pipeline(argv=pipeline_args)
-  (p  # pylint: disable=expression-not-assigned
-   | 'read' >> ReadFromText(known_args.input, coder=JsonCoder())
-   | 'points' >> beam.FlatMap(compute_points)
-   | beam.CombinePerKey(sum)
-   | 'write' >> WriteToText(known_args.output, coder=JsonCoder()))
-  p.run()
+
+  with beam.Pipeline(options=pipeline_options) as p:
+    (p  # pylint: disable=expression-not-assigned
+     | 'read' >> ReadFromText(known_args.input, coder=JsonCoder())
+     | 'points' >> beam.FlatMap(compute_points)
+     | beam.CombinePerKey(sum)
+     | 'write' >> WriteToText(known_args.output, coder=JsonCoder()))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/coders_test.py b/sdks/python/apache_beam/examples/cookbook/coders_test.py
index f71dad8ed80f..988d3c9d25e1 100644
--- a/sdks/python/apache_beam/examples/cookbook/coders_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/coders_test.py
@@ -35,13 +35,13 @@ class CodersTest(unittest.TestCase):
       {'host': ['Brasil', 1], 'guest': ['Italy', 0]}]
 
   def test_compute_points(self):
-    p = TestPipeline()
-    records = p | 'create' >> beam.Create(self.SAMPLE_RECORDS)
-    result = (records
-              | 'points' >> beam.FlatMap(coders.compute_points)
-              | beam.CombinePerKey(sum))
-    assert_that(result, equal_to([('Italy', 0), ('Brasil', 6), ('Germany', 3)]))
-    p.run()
+    with TestPipeline() as p:
+      records = p | 'create' >> beam.Create(self.SAMPLE_RECORDS)
+      result = (records
+                | 'points' >> beam.FlatMap(coders.compute_points)
+                | beam.CombinePerKey(sum))
+      assert_that(result,
+                  equal_to([('Italy', 0), ('Brasil', 6), ('Germany', 3)]))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
index 609f2cd87181..aee69d23ff65 100644
--- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
+++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py
@@ -47,11 +47,10 @@ def expand(self, pcoll):
 def run_count1(known_args, options):
   """Runs the first example pipeline."""
   logging.info('Running first pipeline')
-  p = beam.Pipeline(options=options)
-  (p | beam.io.ReadFromText(known_args.input)
-   | Count1()
-   | beam.io.WriteToText(known_args.output))
-  p.run().wait_until_finish()
+  with beam.Pipeline(options=options) as p:
+    (p | beam.io.ReadFromText(known_args.input)
+     | Count1()
+     | beam.io.WriteToText(known_args.output))
 
 
 @beam.ptransform_fn
@@ -66,11 +65,10 @@ def Count2(pcoll):  # pylint: disable=invalid-name
 def run_count2(known_args, options):
   """Runs the second example pipeline."""
   logging.info('Running second pipeline')
-  p = beam.Pipeline(options=options)
-  (p | ReadFromText(known_args.input)
-   | Count2()  # pylint: disable=no-value-for-parameter
-   | WriteToText(known_args.output))
-  p.run().wait_until_finish()
+  with beam.Pipeline(options=options) as p:
+    (p | ReadFromText(known_args.input)
+     | Count2()  # pylint: disable=no-value-for-parameter
+     | WriteToText(known_args.output))
 
 
 @beam.ptransform_fn
@@ -93,11 +91,10 @@ def Count3(pcoll, factor=1):  # pylint: disable=invalid-name
 def run_count3(known_args, options):
   """Runs the third example pipeline."""
   logging.info('Running third pipeline')
-  p = beam.Pipeline(options=options)
-  (p | ReadFromText(known_args.input)
-   | Count3(2)  # pylint: disable=no-value-for-parameter
-   | WriteToText(known_args.output))
-  p.run()
+  with beam.Pipeline(options=options) as p:
+    (p | ReadFromText(known_args.input)
+     | Count3(2)  # pylint: disable=no-value-for-parameter
+     | WriteToText(known_args.output))
 
 
 def get_args(argv):
diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
index c7c6dbabd5e0..7aaccb4ac1c0 100644
--- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py
@@ -40,12 +40,11 @@ def test_count3(self):
     self.run_pipeline(custom_ptransform.Count3(factor), factor=factor)
 
   def run_pipeline(self, count_implementation, factor=1):
-    p = TestPipeline()
-    words = p | beam.Create(['CAT', 'DOG', 'CAT', 'CAT', 'DOG'])
-    result = words | count_implementation
-    assert_that(
-        result, equal_to([('CAT', (3 * factor)), ('DOG', (2 * factor))]))
-    p.run()
+    with TestPipeline() as p:
+      words = p | beam.Create(['CAT', 'DOG', 'CAT', 'CAT', 'DOG'])
+      result = words | count_implementation
+      assert_that(
+          result, equal_to([('CAT', (3 * factor)), ('DOG', (2 * factor))]))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
index 411feb8058a7..7161cff1c255 100644
--- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
+++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py
@@ -135,18 +135,15 @@ def make_entity(self, content):
 
 def write_to_datastore(project, user_options, pipeline_options):
   """Creates a pipeline that writes entities to Cloud Datastore."""
-  p = beam.Pipeline(options=pipeline_options)
-
-  # pylint: disable=expression-not-assigned
-  (p
-   | 'read' >> ReadFromText(user_options.input)
-   | 'create entity' >> beam.Map(
-       EntityWrapper(user_options.namespace, user_options.kind,
-                     user_options.ancestor).make_entity)
-   | 'write to datastore' >> WriteToDatastore(project))
+  with beam.Pipeline(options=pipeline_options) as p:
 
-  # Actually run the pipeline (all operations above are deferred).
-  p.run().wait_until_finish()
+    # pylint: disable=expression-not-assigned
+    (p
+     | 'read' >> ReadFromText(user_options.input)
+     | 'create entity' >> beam.Map(
+         EntityWrapper(user_options.namespace, user_options.kind,
+                       user_options.ancestor).make_entity)
+     | 'write to datastore' >> WriteToDatastore(project))
 
 
 def make_ancestor_query(kind, namespace, ancestor):
@@ -196,7 +193,6 @@ def read_from_datastore(project, user_options, pipeline_options):
   output | 'write' >> beam.io.WriteToText(file_path_prefix=user_options.output,
                                           num_shards=user_options.num_shards)
 
-  # Actually run the pipeline (all operations above are deferred).
   result = p.run()
   # Wait until completion, main thread would access post-completion job results.
   result.wait_until_finish()
diff --git a/sdks/python/apache_beam/examples/cookbook/filters.py b/sdks/python/apache_beam/examples/cookbook/filters.py
index 374001cd83fe..1fbf763e5005 100644
--- a/sdks/python/apache_beam/examples/cookbook/filters.py
+++ b/sdks/python/apache_beam/examples/cookbook/filters.py
@@ -86,20 +86,17 @@ def run(argv=None):
                       help='Numeric value of month to filter on.')
   known_args, pipeline_args = parser.parse_known_args(argv)
 
-  p = beam.Pipeline(argv=pipeline_args)
+  with beam.Pipeline(argv=pipeline_args) as p:
 
-  input_data = p | beam.io.Read(beam.io.BigQuerySource(known_args.input))
+    input_data = p | beam.io.Read(beam.io.BigQuerySource(known_args.input))
 
-  # pylint: disable=expression-not-assigned
-  (filter_cold_days(input_data, known_args.month_filter)
-   | 'SaveToBQ' >> beam.io.Write(beam.io.BigQuerySink(
-       known_args.output,
-       schema='year:INTEGER,month:INTEGER,day:INTEGER,mean_temp:FLOAT',
-       create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
-       write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)))
-
-  # Actually run the pipeline (all operations above are deferred).
-  p.run()
+    # pylint: disable=expression-not-assigned
+    (filter_cold_days(input_data, known_args.month_filter)
+     | 'SaveToBQ' >> beam.io.Write(beam.io.BigQuerySink(
+         known_args.output,
+         schema='year:INTEGER,month:INTEGER,day:INTEGER,mean_temp:FLOAT',
+         create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+         write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
index 6bdadae9f3d1..9c0d04b816ac 100644
--- a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
+++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py
@@ -95,28 +95,27 @@ def run(args=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  # Register the custom coder for the Player class, so that it will be used in
-  # the computation.
-  coders.registry.register_coder(Player, PlayerCoder)
-
-  (p  # pylint: disable=expression-not-assigned
-   | ReadFromText(known_args.input)
-   # The get_players function is annotated with a type hint above, so the type
-   # system knows the output type of the following operation is a key-value pair
-   # of a Player and an int. Please see the documentation for details on
-   # types that are inferred automatically as well as other ways to specify
-   # type hints.
-   | beam.Map(get_players)
-   # The output type hint of the previous step is used to infer that the key
-   # type of the following operation is the Player type. Since a custom coder
-   # is registered for the Player class above, a PlayerCoder will be used to
-   # encode Player objects as keys for this combine operation.
-   | beam.CombinePerKey(sum)
-   | beam.Map(lambda (k, v): '%s,%d' % (k.name, v))
-   | WriteToText(known_args.output))
-  return p.run()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    # Register the custom coder for the Player class, so that it will be used in
+    # the computation.
+    coders.registry.register_coder(Player, PlayerCoder)
+
+    (p  # pylint: disable=expression-not-assigned
+     | ReadFromText(known_args.input)
+     # The get_players function is annotated with a type hint above, so the type
+     # system knows the output type of the following operation is a key-value
+     # pair of a Player and an int. Please see the documentation for details on
+     # types that are inferred automatically as well as other ways to specify
+     # type hints.
+     | beam.Map(get_players)
+     # The output type hint of the previous step is used to infer that the key
+     # type of the following operation is the Player type. Since a custom coder
+     # is registered for the Player class above, a PlayerCoder will be used to
+     # encode Player objects as keys for this combine operation.
+     | beam.CombinePerKey(sum)
+     | beam.Map(lambda (k, v): '%s,%d' % (k.name, v))
+     | WriteToText(known_args.output))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py
index 4e8796647da0..268ba8d355f2 100644
--- a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py
@@ -50,7 +50,7 @@ def test_basics_with_type_check(self):
     temp_path = self.create_temp_file(self.SAMPLE_RECORDS)
     group_with_coder.run([
         '--input=%s*' % temp_path,
-        '--output=%s.result' % temp_path]).wait_until_finish()
+        '--output=%s.result' % temp_path])
     # Parse result file and compare.
     results = []
     with open(temp_path + '.result-00000-of-00001') as result_file:
@@ -71,7 +71,7 @@ def test_basics_without_type_check(self):
     group_with_coder.run([
         '--no_pipeline_type_check',
         '--input=%s*' % temp_path,
-        '--output=%s.result' % temp_path]).wait_until_finish()
+        '--output=%s.result' % temp_path])
     # Parse result file and compare.
     results = []
     with open(temp_path + '.result-00000-of-00001') as result_file:
diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
index 4f53c615ba97..9acdd9073478 100644
--- a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
+++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py
@@ -70,64 +70,63 @@ def run(argv=None, assert_results=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  # Helper: read a tab-separated key-value mapping from a text file, escape all
-  # quotes/backslashes, and convert it a PCollection of (key, value) pairs.
-  def read_kv_textfile(label, textfile):
-    return (p
-            | 'Read: %s' % label >> ReadFromText(textfile)
-            | 'Backslash: %s' % label >> beam.Map(
-                lambda x: re.sub(r'\\', r'\\\\', x))
-            | 'EscapeQuotes: %s' % label >> beam.Map(
-                lambda x: re.sub(r'"', r'\"', x))
-            | 'Split: %s' % label >> beam.Map(
-                lambda x: re.split(r'\t+', x, 1)))
-
-  # Read input databases.
-  email = read_kv_textfile('email', known_args.input_email)
-  phone = read_kv_textfile('phone', known_args.input_phone)
-  snailmail = read_kv_textfile('snailmail', known_args.input_snailmail)
-
-  # Group together all entries under the same name.
-  grouped = (email, phone, snailmail) | 'group_by_name' >> beam.CoGroupByKey()
-
-  # Prepare tab-delimited output; something like this:
-  # "name"<TAB>"email_1,email_2"<TAB>"phone"<TAB>"first_snailmail_only"
-  tsv_lines = grouped | beam.Map(
-      lambda (name, (email, phone, snailmail)): '\t'.join(
-          ['"%s"' % name,
-           '"%s"' % ','.join(email),
-           '"%s"' % ','.join(phone),
-           '"%s"' % next(iter(snailmail), '')]))
-
-  # Compute some stats about our database of people.
-  luddites = grouped | beam.Filter(  # People without email.
-      lambda (name, (email, phone, snailmail)): not next(iter(email), None))
-  writers = grouped | beam.Filter(   # People without phones.
-      lambda (name, (email, phone, snailmail)): not next(iter(phone), None))
-  nomads = grouped | beam.Filter(    # People without addresses.
-      lambda (name, (email, phone, snailmail)): not next(iter(snailmail), None))
-
-  num_luddites = luddites | 'Luddites' >> beam.combiners.Count.Globally()
-  num_writers = writers | 'Writers' >> beam.combiners.Count.Globally()
-  num_nomads = nomads | 'Nomads' >> beam.combiners.Count.Globally()
-
-  # Write tab-delimited output.
-  # pylint: disable=expression-not-assigned
-  tsv_lines | 'WriteTsv' >> WriteToText(known_args.output_tsv)
-
-  # TODO(silviuc): Move the assert_results logic to the unit test.
-  if assert_results is not None:
-    expected_luddites, expected_writers, expected_nomads = assert_results
-    assert_that(num_luddites, equal_to([expected_luddites]),
-                label='assert:luddites')
-    assert_that(num_writers, equal_to([expected_writers]),
-                label='assert:writers')
-    assert_that(num_nomads, equal_to([expected_nomads]),
-                label='assert:nomads')
-  # Execute pipeline.
-  return p.run()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    # Helper: read a tab-separated key-value mapping from a text file,
+    # escape all quotes/backslashes, and convert it a PCollection of
+    # (key, value) pairs.
+    def read_kv_textfile(label, textfile):
+      return (p
+              | 'Read: %s' % label >> ReadFromText(textfile)
+              | 'Backslash: %s' % label >> beam.Map(
+                  lambda x: re.sub(r'\\', r'\\\\', x))
+              | 'EscapeQuotes: %s' % label >> beam.Map(
+                  lambda x: re.sub(r'"', r'\"', x))
+              | 'Split: %s' % label >> beam.Map(
+                  lambda x: re.split(r'\t+', x, 1)))
+
+    # Read input databases.
+    email = read_kv_textfile('email', known_args.input_email)
+    phone = read_kv_textfile('phone', known_args.input_phone)
+    snailmail = read_kv_textfile('snailmail', known_args.input_snailmail)
+
+    # Group together all entries under the same name.
+    grouped = (email, phone, snailmail) | 'group_by_name' >> beam.CoGroupByKey()
+
+    # Prepare tab-delimited output; something like this:
+    # "name"<TAB>"email_1,email_2"<TAB>"phone"<TAB>"first_snailmail_only"
+    tsv_lines = grouped | beam.Map(
+        lambda (name, (email, phone, snailmail)): '\t'.join(
+            ['"%s"' % name,
+             '"%s"' % ','.join(email),
+             '"%s"' % ','.join(phone),
+             '"%s"' % next(iter(snailmail), '')]))
+
+    # Compute some stats about our database of people.
+    luddites = grouped | beam.Filter(  # People without email.
+        lambda (name, (email, phone, snailmail)): not next(iter(email), None))
+    writers = grouped | beam.Filter(   # People without phones.
+        lambda (name, (email, phone, snailmail)): not next(iter(phone), None))
+    nomads = grouped | beam.Filter(    # People without addresses.
+        lambda (name, (e, p, snailmail)): not next(iter(snailmail), None))
+
+    num_luddites = luddites | 'Luddites' >> beam.combiners.Count.Globally()
+    num_writers = writers | 'Writers' >> beam.combiners.Count.Globally()
+    num_nomads = nomads | 'Nomads' >> beam.combiners.Count.Globally()
+
+    # Write tab-delimited output.
+    # pylint: disable=expression-not-assigned
+    tsv_lines | 'WriteTsv' >> WriteToText(known_args.output_tsv)
+
+    # TODO(silviuc): Move the assert_results logic to the unit test.
+    if assert_results is not None:
+      expected_luddites, expected_writers, expected_nomads = assert_results
+      assert_that(num_luddites, equal_to([expected_luddites]),
+                  label='assert:luddites')
+      assert_that(num_writers, equal_to([expected_writers]),
+                  label='assert:writers')
+      assert_that(num_nomads, equal_to([expected_nomads]),
+                  label='assert:nomads')
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py
index 09f71d389696..b3be0ddb0853 100644
--- a/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py
@@ -107,13 +107,12 @@ def test_mergecontacts(self):
 
     result_prefix = self.create_temp_file('')
 
-    result = mergecontacts.run([
+    mergecontacts.run([
         '--input_email=%s' % path_email,
         '--input_phone=%s' % path_phone,
         '--input_snailmail=%s' % path_snailmail,
         '--output_tsv=%s.tsv' % result_prefix,
         '--output_stats=%s.stats' % result_prefix], assert_results=(2, 1, 3))
-    result.wait_until_finish()
 
     with open('%s.tsv-00000-of-00001' % result_prefix) as f:
       contents = f.read()
diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
index 9759f4821045..2316c6611c06 100644
--- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
+++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
@@ -141,43 +141,41 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  lines = p | ReadFromText(known_args.input)
-
-  # with_outputs allows accessing the explicitly tagged outputs of a DoFn.
-  split_lines_result = (lines
-                        | beam.ParDo(SplitLinesToWordsFn()).with_outputs(
-                            SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS,
-                            SplitLinesToWordsFn.OUTPUT_TAG_CHARACTER_COUNT,
-                            main='words'))
-
-  # split_lines_result is an object of type DoOutputsTuple. It supports
-  # accessing result in alternative ways.
-  words, _, _ = split_lines_result
-  short_words = split_lines_result[
-      SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS]
-  character_count = split_lines_result.tag_character_count
-
-  # pylint: disable=expression-not-assigned
-  (character_count
-   | 'pair_with_key' >> beam.Map(lambda x: ('chars_temp_key', x))
-   | beam.GroupByKey()
-   | 'count chars' >> beam.Map(lambda (_, counts): sum(counts))
-   | 'write chars' >> WriteToText(known_args.output + '-chars'))
-
-  # pylint: disable=expression-not-assigned
-  (short_words
-   | 'count short words' >> CountWords()
-   | 'write short words' >> WriteToText(
-       known_args.output + '-short-words'))
-
-  # pylint: disable=expression-not-assigned
-  (words
-   | 'count words' >> CountWords()
-   | 'write words' >> WriteToText(known_args.output + '-words'))
-
-  return p.run()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    lines = p | ReadFromText(known_args.input)
+
+    # with_outputs allows accessing the explicitly tagged outputs of a DoFn.
+    split_lines_result = (lines
+                          | beam.ParDo(SplitLinesToWordsFn()).with_outputs(
+                              SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS,
+                              SplitLinesToWordsFn.OUTPUT_TAG_CHARACTER_COUNT,
+                              main='words'))
+
+    # split_lines_result is an object of type DoOutputsTuple. It supports
+    # accessing result in alternative ways.
+    words, _, _ = split_lines_result
+    short_words = split_lines_result[
+        SplitLinesToWordsFn.OUTPUT_TAG_SHORT_WORDS]
+    character_count = split_lines_result.tag_character_count
+
+    # pylint: disable=expression-not-assigned
+    (character_count
+     | 'pair_with_key' >> beam.Map(lambda x: ('chars_temp_key', x))
+     | beam.GroupByKey()
+     | 'count chars' >> beam.Map(lambda (_, counts): sum(counts))
+     | 'write chars' >> WriteToText(known_args.output + '-chars'))
+
+    # pylint: disable=expression-not-assigned
+    (short_words
+     | 'count short words' >> CountWords()
+     | 'write short words' >> WriteToText(
+         known_args.output + '-short-words'))
+
+    # pylint: disable=expression-not-assigned
+    (words
+     | 'count words' >> CountWords()
+     | 'write words' >> WriteToText(known_args.output + '-words'))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py
index 2c9111c678bd..3ddd668599d0 100644
--- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py
@@ -52,7 +52,7 @@ def test_multiple_output_pardo(self):
 
     multiple_output_pardo.run([
         '--input=%s*' % temp_path,
-        '--output=%s' % result_prefix]).wait_until_finish()
+        '--output=%s' % result_prefix])
 
     expected_char_count = len(''.join(self.SAMPLE_TEXT.split('\n')))
     with open(result_prefix + '-chars-00000-of-00001') as f:
diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py
index 72595723de45..70929e9e8fae 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets.py
@@ -147,18 +147,15 @@ def _add_argparse_args(cls, parser):
   pipeline_options = PipelineOptions(argv)
   my_options = pipeline_options.view_as(MyOptions)
 
-  p = beam.Pipeline(options=pipeline_options)
-
-  (p
-   | beam.io.ReadFromText(my_options.input)
-   | beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
-   | beam.Map(lambda x: (x, 1))
-   | beam.combiners.Count.PerKey()
-   | beam.io.WriteToText(my_options.output))
-
-  result = p.run()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    (p
+     | beam.io.ReadFromText(my_options.input)
+     | beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+     | beam.Map(lambda x: (x, 1))
+     | beam.combiners.Count.PerKey()
+     | beam.io.WriteToText(my_options.output))
   # [END model_pipelines]
-  result.wait_until_finish()
 
 
 def model_pcollection(argv):
@@ -178,21 +175,18 @@ def _add_argparse_args(cls, parser):
   my_options = pipeline_options.view_as(MyOptions)
 
   # [START model_pcollection]
-  p = beam.Pipeline(options=pipeline_options)
+  with beam.Pipeline(options=pipeline_options) as p:
 
-  lines = (p
-           | beam.Create([
-               'To be, or not to be: that is the question: ',
-               'Whether \'tis nobler in the mind to suffer ',
-               'The slings and arrows of outrageous fortune, ',
-               'Or to take arms against a sea of troubles, ']))
-  # [END model_pcollection]
+    lines = (p
+             | beam.Create([
+                 'To be, or not to be: that is the question: ',
+                 'Whether \'tis nobler in the mind to suffer ',
+                 'The slings and arrows of outrageous fortune, ',
+                 'Or to take arms against a sea of troubles, ']))
+    # [END model_pcollection]
 
-  (lines
-   | beam.io.WriteToText(my_options.output))
-
-  result = p.run()
-  result.wait_until_finish()
+    (lines
+     | beam.io.WriteToText(my_options.output))
 
 
 def pipeline_options_remote(argv):
@@ -297,12 +291,10 @@ def pipeline_options_command_line(argv):
   known_args, pipeline_args = parser.parse_known_args(argv)
 
   # Create the Pipeline with remaining arguments.
-  p = beam.Pipeline(argv=pipeline_args)
-  lines = p | 'ReadFromText' >> beam.io.ReadFromText(known_args.input)
-  lines | 'WriteToText' >> beam.io.WriteToText(known_args.output)
-  # [END pipeline_options_command_line]
-
-  p.run().wait_until_finish()
+  with beam.Pipeline(argv=pipeline_args) as p:
+    lines = p | 'ReadFromText' >> beam.io.ReadFromText(known_args.input)
+    lines | 'WriteToText' >> beam.io.WriteToText(known_args.output)
+    # [END pipeline_options_command_line]
 
 
 def pipeline_logging(lines, output):
@@ -329,13 +321,11 @@ def process(self, element):
   # Remaining WordCount example code ...
   # [END pipeline_logging]
 
-  p = TestPipeline()  # Use TestPipeline for testing.
-  (p
-   | beam.Create(lines)
-   | beam.ParDo(ExtractWordsFn())
-   | beam.io.WriteToText(output))
-
-  p.run()
+  with TestPipeline() as p:  # Use TestPipeline for testing.
+    (p
+     | beam.Create(lines)
+     | beam.ParDo(ExtractWordsFn())
+     | beam.io.WriteToText(output))
 
 
 def pipeline_monitoring(renames):
@@ -385,20 +375,19 @@ def expand(self, pcoll):
 
   pipeline_options = PipelineOptions()
   options = pipeline_options.view_as(WordCountOptions)
-  p = TestPipeline()  # Use TestPipeline for testing.
+  with TestPipeline() as p:  # Use TestPipeline for testing.
 
-  # [START pipeline_monitoring_execution]
-  (p
-   # Read the lines of the input text.
-   | 'ReadLines' >> beam.io.ReadFromText(options.input)
-   # Count the words.
-   | CountWords()
-   # Write the formatted word counts to output.
-   | 'WriteCounts' >> beam.io.WriteToText(options.output))
-  # [END pipeline_monitoring_execution]
+    # [START pipeline_monitoring_execution]
+    (p
+     # Read the lines of the input text.
+     | 'ReadLines' >> beam.io.ReadFromText(options.input)
+     # Count the words.
+     | CountWords()
+     # Write the formatted word counts to output.
+     | 'WriteCounts' >> beam.io.WriteToText(options.output))
+    # [END pipeline_monitoring_execution]
 
-  p.visit(SnippetUtils.RenameFiles(renames))
-  p.run()
+    p.visit(SnippetUtils.RenameFiles(renames))
 
 
 def examples_wordcount_minimal(renames):
@@ -478,40 +467,39 @@ def _add_argparse_args(cls, parser):
                           default='gs://my-bucket/input')
 
   options = PipelineOptions(argv)
-  p = beam.Pipeline(options=options)
-  # [END examples_wordcount_wordcount_options]
+  with beam.Pipeline(options=options) as p:
+    # [END examples_wordcount_wordcount_options]
 
-  lines = p | beam.io.ReadFromText(
-      'gs://dataflow-samples/shakespeare/kinglear.txt')
+    lines = p | beam.io.ReadFromText(
+        'gs://dataflow-samples/shakespeare/kinglear.txt')
 
-  # [START examples_wordcount_wordcount_composite]
-  class CountWords(beam.PTransform):
+    # [START examples_wordcount_wordcount_composite]
+    class CountWords(beam.PTransform):
 
-    def expand(self, pcoll):
-      return (pcoll
-              # Convert lines of text into individual words.
-              | 'ExtractWords' >> beam.FlatMap(
-                  lambda x: re.findall(r'[A-Za-z\']+', x))
+      def expand(self, pcoll):
+        return (pcoll
+                # Convert lines of text into individual words.
+                | 'ExtractWords' >> beam.FlatMap(
+                    lambda x: re.findall(r'[A-Za-z\']+', x))
 
-              # Count the number of times each word occurs.
-              | beam.combiners.Count.PerElement())
+                # Count the number of times each word occurs.
+                | beam.combiners.Count.PerElement())
 
-  counts = lines | CountWords()
-  # [END examples_wordcount_wordcount_composite]
+    counts = lines | CountWords()
+    # [END examples_wordcount_wordcount_composite]
 
-  # [START examples_wordcount_wordcount_dofn]
-  class FormatAsTextFn(beam.DoFn):
+    # [START examples_wordcount_wordcount_dofn]
+    class FormatAsTextFn(beam.DoFn):
 
-    def process(self, element):
-      word, count = element
-      yield '%s: %s' % (word, count)
+      def process(self, element):
+        word, count = element
+        yield '%s: %s' % (word, count)
 
-  formatted = counts | beam.ParDo(FormatAsTextFn())
-  # [END examples_wordcount_wordcount_dofn]
+    formatted = counts | beam.ParDo(FormatAsTextFn())
+    # [END examples_wordcount_wordcount_dofn]
 
-  formatted |  beam.io.WriteToText('gs://my-bucket/counts.txt')
-  p.visit(SnippetUtils.RenameFiles(renames))
-  p.run().wait_until_finish()
+    formatted |  beam.io.WriteToText('gs://my-bucket/counts.txt')
+    p.visit(SnippetUtils.RenameFiles(renames))
 
 
 def examples_wordcount_debugging(renames):
@@ -558,27 +546,27 @@ def process(self, element):
   # [END example_wordcount_debugging_logging]
   # [END example_wordcount_debugging_aggregators]
 
-  p = TestPipeline()  # Use TestPipeline for testing.
-  filtered_words = (
-      p
-      | beam.io.ReadFromText(
-          'gs://dataflow-samples/shakespeare/kinglear.txt')
-      | 'ExtractWords' >> beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
-      | beam.combiners.Count.PerElement()
-      | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
+  with TestPipeline() as p:  # Use TestPipeline for testing.
+    filtered_words = (
+        p
+        | beam.io.ReadFromText(
+            'gs://dataflow-samples/shakespeare/kinglear.txt')
+        | 'ExtractWords' >> beam.FlatMap(
+            lambda x: re.findall(r'[A-Za-z\']+', x))
+        | beam.combiners.Count.PerElement()
+        | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
 
-  # [START example_wordcount_debugging_assert]
-  beam.testing.util.assert_that(
-      filtered_words, beam.testing.util.equal_to(
-          [('Flourish', 3), ('stomach', 1)]))
-  # [END example_wordcount_debugging_assert]
+    # [START example_wordcount_debugging_assert]
+    beam.testing.util.assert_that(
+        filtered_words, beam.testing.util.equal_to(
+            [('Flourish', 3), ('stomach', 1)]))
+    # [END example_wordcount_debugging_assert]
 
-  output = (filtered_words
-            | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
-            | 'Write' >> beam.io.WriteToText('gs://my-bucket/counts.txt'))
+    output = (filtered_words
+              | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
+              | 'Write' >> beam.io.WriteToText('gs://my-bucket/counts.txt'))
 
-  p.visit(SnippetUtils.RenameFiles(renames))
-  p.run()
+    p.visit(SnippetUtils.RenameFiles(renames))
 
 
 import apache_beam as beam
@@ -659,16 +647,14 @@ def model_custom_source(count):
 
   # Using the source in an example pipeline.
   # [START model_custom_source_use_new_source]
-  p = beam.Pipeline(options=PipelineOptions())
-  numbers = p | 'ProduceNumbers' >> beam.io.Read(CountingSource(count))
-  # [END model_custom_source_use_new_source]
+  with beam.Pipeline(options=PipelineOptions()) as p:
+    numbers = p | 'ProduceNumbers' >> beam.io.Read(CountingSource(count))
+    # [END model_custom_source_use_new_source]
 
-  lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
-  assert_that(
-      lines, equal_to(
-          ['line ' + str(number) for number in range(0, count)]))
-
-  p.run().wait_until_finish()
+    lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
+    assert_that(
+        lines, equal_to(
+            ['line ' + str(number) for number in range(0, count)]))
 
   # We recommend users to start Source classes with an underscore to discourage
   # using the Source class directly when a PTransform for the source is
@@ -796,14 +782,12 @@ def close(self):
 
   # Using the new sink in an example pipeline.
   # [START model_custom_sink_use_new_sink]
-  p = beam.Pipeline(options=PipelineOptions())
-  kvs = p | 'CreateKVs' >> beam.Create(KVs)
+  with beam.Pipeline(options=PipelineOptions()) as p:
+    kvs = p | 'CreateKVs' >> beam.Create(KVs)
 
-  kvs | 'WriteToSimpleKV' >> beam.io.Write(
-      SimpleKVSink('http://url_to_simple_kv/', final_table_name))
-  # [END model_custom_sink_use_new_sink]
-
-  p.run().wait_until_finish()
+    kvs | 'WriteToSimpleKV' >> beam.io.Write(
+        SimpleKVSink('http://url_to_simple_kv/', final_table_name))
+    # [END model_custom_sink_use_new_sink]
 
   # We recommend users to start Sink class names with an underscore to
   # discourage using the Sink class directly when a PTransform for the sink is
@@ -828,13 +812,11 @@ def expand(self, pcoll):
   final_table_name = final_table_name_with_ptransform
 
   # [START model_custom_sink_use_ptransform]
-  p = beam.Pipeline(options=PipelineOptions())
-  kvs = p | 'CreateKVs' >> beam.core.Create(KVs)
-  kvs | 'WriteToSimpleKV' >> WriteToKVSink(
-      'http://url_to_simple_kv/', final_table_name)
-  # [END model_custom_sink_use_ptransform]
-
-  p.run().wait_until_finish()
+  with beam.Pipeline(options=PipelineOptions()) as p:
+    kvs = p | 'CreateKVs' >> beam.core.Create(KVs)
+    kvs | 'WriteToSimpleKV' >> WriteToKVSink(
+        'http://url_to_simple_kv/', final_table_name)
+    # [END model_custom_sink_use_ptransform]
 
 
 def model_textio(renames):
@@ -847,37 +829,35 @@ def filter_words(x):
   from apache_beam.options.pipeline_options import PipelineOptions
 
   # [START model_textio_read]
-  p = beam.Pipeline(options=PipelineOptions())
-  # [START model_pipelineio_read]
-  lines = p | 'ReadFromText' >> beam.io.ReadFromText('path/to/input-*.csv')
-  # [END model_pipelineio_read]
-  # [END model_textio_read]
-
-  # [START model_textio_write]
-  filtered_words = lines | 'FilterWords' >> beam.FlatMap(filter_words)
-  # [START model_pipelineio_write]
-  filtered_words | 'WriteToText' >> beam.io.WriteToText(
-      '/path/to/numbers', file_name_suffix='.csv')
-  # [END model_pipelineio_write]
-  # [END model_textio_write]
+  with beam.Pipeline(options=PipelineOptions()) as p:
+    # [START model_pipelineio_read]
+    lines = p | 'ReadFromText' >> beam.io.ReadFromText('path/to/input-*.csv')
+    # [END model_pipelineio_read]
+    # [END model_textio_read]
 
-  p.visit(SnippetUtils.RenameFiles(renames))
-  p.run().wait_until_finish()
+    # [START model_textio_write]
+    filtered_words = lines | 'FilterWords' >> beam.FlatMap(filter_words)
+    # [START model_pipelineio_write]
+    filtered_words | 'WriteToText' >> beam.io.WriteToText(
+        '/path/to/numbers', file_name_suffix='.csv')
+    # [END model_pipelineio_write]
+    # [END model_textio_write]
+
+    p.visit(SnippetUtils.RenameFiles(renames))
 
 
 def model_textio_compressed(renames, expected):
   """Using a Read Transform to read compressed text files."""
-  p = TestPipeline()
+  with TestPipeline() as p:
 
-  # [START model_textio_write_compressed]
-  lines = p | 'ReadFromText' >> beam.io.ReadFromText(
-      '/path/to/input-*.csv.gz',
-      compression_type=beam.io.filesystem.CompressionTypes.GZIP)
-  # [END model_textio_write_compressed]
+    # [START model_textio_write_compressed]
+    lines = p | 'ReadFromText' >> beam.io.ReadFromText(
+        '/path/to/input-*.csv.gz',
+        compression_type=beam.io.filesystem.CompressionTypes.GZIP)
+    # [END model_textio_write_compressed]
 
-  assert_that(lines, equal_to(expected))
-  p.visit(SnippetUtils.RenameFiles(renames))
-  p.run().wait_until_finish()
+    assert_that(lines, equal_to(expected))
+    p.visit(SnippetUtils.RenameFiles(renames))
 
 
 def model_datastoreio():
@@ -987,43 +967,40 @@ def expand(self, pcoll):
   # [END composite_ptransform_apply_method]
   # [END composite_transform_example]
 
-  p = TestPipeline()  # Use TestPipeline for testing.
-  (p
-   | beam.Create(contents)
-   | CountWords()
-   | beam.io.WriteToText(output_path))
-  p.run()
+  with TestPipeline() as p:  # Use TestPipeline for testing.
+    (p
+     | beam.Create(contents)
+     | CountWords()
+     | beam.io.WriteToText(output_path))
 
 
 def model_multiple_pcollections_flatten(contents, output_path):
   """Merging a PCollection with Flatten."""
   some_hash_fn = lambda s: ord(s[0])
-  import apache_beam as beam
-  p = TestPipeline()  # Use TestPipeline for testing.
   partition_fn = lambda element, partitions: some_hash_fn(element) % partitions
-
-  # Partition into deciles
-  partitioned = p | beam.Create(contents) | beam.Partition(partition_fn, 3)
-  pcoll1 = partitioned[0]
-  pcoll2 = partitioned[1]
-  pcoll3 = partitioned[2]
-
-  # Flatten them back into 1
-
-  # A collection of PCollection objects can be represented simply
-  # as a tuple (or list) of PCollections.
-  # (The SDK for Python has no separate type to store multiple
-  # PCollection objects, whether containing the same or different
-  # types.)
-  # [START model_multiple_pcollections_flatten]
-  merged = (
-      (pcoll1, pcoll2, pcoll3)
-      # A list of tuples can be "piped" directly into a Flatten transform.
-      | beam.Flatten())
-  # [END model_multiple_pcollections_flatten]
-  merged | beam.io.WriteToText(output_path)
-
-  p.run()
+  import apache_beam as beam
+  with TestPipeline() as p:  # Use TestPipeline for testing.
+
+    # Partition into deciles
+    partitioned = p | beam.Create(contents) | beam.Partition(partition_fn, 3)
+    pcoll1 = partitioned[0]
+    pcoll2 = partitioned[1]
+    pcoll3 = partitioned[2]
+
+    # Flatten them back into 1
+
+    # A collection of PCollection objects can be represented simply
+    # as a tuple (or list) of PCollections.
+    # (The SDK for Python has no separate type to store multiple
+    # PCollection objects, whether containing the same or different
+    # types.)
+    # [START model_multiple_pcollections_flatten]
+    merged = (
+        (pcoll1, pcoll2, pcoll3)
+        # A list of tuples can be "piped" directly into a Flatten transform.
+        | beam.Flatten())
+    # [END model_multiple_pcollections_flatten]
+    merged | beam.io.WriteToText(output_path)
 
 
 def model_multiple_pcollections_partition(contents, output_path):
@@ -1034,25 +1011,23 @@ def get_percentile(i):
     """Assume i in [0,100)."""
     return i
   import apache_beam as beam
-  p = TestPipeline()  # Use TestPipeline for testing.
+  with TestPipeline() as p:  # Use TestPipeline for testing.
 
-  students = p | beam.Create(contents)
+    students = p | beam.Create(contents)
 
-  # [START model_multiple_pcollections_partition]
-  def partition_fn(student, num_partitions):
-    return int(get_percentile(student) * num_partitions / 100)
+    # [START model_multiple_pcollections_partition]
+    def partition_fn(student, num_partitions):
+      return int(get_percentile(student) * num_partitions / 100)
 
-  by_decile = students | beam.Partition(partition_fn, 10)
-  # [END model_multiple_pcollections_partition]
-  # [START model_multiple_pcollections_partition_40th]
-  fortieth_percentile = by_decile[4]
-  # [END model_multiple_pcollections_partition_40th]
+    by_decile = students | beam.Partition(partition_fn, 10)
+    # [END model_multiple_pcollections_partition]
+    # [START model_multiple_pcollections_partition_40th]
+    fortieth_percentile = by_decile[4]
+    # [END model_multiple_pcollections_partition_40th]
 
-  ([by_decile[d] for d in xrange(10) if d != 4] + [fortieth_percentile]
-   | beam.Flatten()
-   | beam.io.WriteToText(output_path))
-
-  p.run()
+    ([by_decile[d] for d in xrange(10) if d != 4] + [fortieth_percentile]
+     | beam.Flatten()
+     | beam.io.WriteToText(output_path))
 
 
 def model_group_by_key(contents, output_path):
@@ -1060,58 +1035,56 @@ def model_group_by_key(contents, output_path):
   import re
 
   import apache_beam as beam
-  p = TestPipeline()  # Use TestPipeline for testing.
-  words_and_counts = (
-      p
-      | beam.Create(contents)
-      | beam.FlatMap(lambda x: re.findall(r'\w+', x))
-      | 'one word' >> beam.Map(lambda w: (w, 1)))
-  # GroupByKey accepts a PCollection of (w, 1) and
-  # outputs a PCollection of (w, (1, 1, ...)).
-  # (A key/value pair is just a tuple in Python.)
-  # This is a somewhat forced example, since one could
-  # simply use beam.combiners.Count.PerElement here.
-  # [START model_group_by_key_transform]
-  grouped_words = words_and_counts | beam.GroupByKey()
-  # [END model_group_by_key_transform]
-  (grouped_words
-   | 'count words' >> beam.Map(lambda (word, counts): (word, len(counts)))
-   | beam.io.WriteToText(output_path))
-  p.run()
+  with TestPipeline() as p:  # Use TestPipeline for testing.
+    words_and_counts = (
+        p
+        | beam.Create(contents)
+        | beam.FlatMap(lambda x: re.findall(r'\w+', x))
+        | 'one word' >> beam.Map(lambda w: (w, 1)))
+    # GroupByKey accepts a PCollection of (w, 1) and
+    # outputs a PCollection of (w, (1, 1, ...)).
+    # (A key/value pair is just a tuple in Python.)
+    # This is a somewhat forced example, since one could
+    # simply use beam.combiners.Count.PerElement here.
+    # [START model_group_by_key_transform]
+    grouped_words = words_and_counts | beam.GroupByKey()
+    # [END model_group_by_key_transform]
+    (grouped_words
+     | 'count words' >> beam.Map(lambda (word, counts): (word, len(counts)))
+     | beam.io.WriteToText(output_path))
 
 
 def model_co_group_by_key_tuple(email_list, phone_list, output_path):
   """Applying a CoGroupByKey Transform to a tuple."""
   import apache_beam as beam
-  p = TestPipeline()  # Use TestPipeline for testing.
-  # [START model_group_by_key_cogroupbykey_tuple]
-  # Each data set is represented by key-value pairs in separate PCollections.
-  # Both data sets share a common key type (in this example str).
-  # The email_list contains values such as: ('joe', 'joe@example.com') with
-  # multiple possible values for each key.
-  # The phone_list contains values such as: ('mary': '111-222-3333') with
-  # multiple possible values for each key.
-  emails = p | 'email' >> beam.Create(email_list)
-  phones = p | 'phone' >> beam.Create(phone_list)
-  # The result PCollection contains one key-value element for each key in the
-  # input PCollections. The key of the pair will be the key from the input and
-  # the value will be a dictionary with two entries: 'emails' - an iterable of
-  # all values for the current key in the emails PCollection and 'phones': an
-  # iterable of all values for the current key in the phones PCollection.
-  # For instance, if 'emails' contained ('joe', 'joe@example.com') and
-  # ('joe', 'joe@gmail.com'), then 'result' will contain the element
-  # ('joe', {'emails': ['joe@example.com', 'joe@gmail.com'], 'phones': ...})
-  result = {'emails': emails, 'phones': phones} | beam.CoGroupByKey()
-
-  def join_info((name, info)):
-    return '; '.join(['%s' % name,
-                      '%s' % ','.join(info['emails']),
-                      '%s' % ','.join(info['phones'])])
-
-  contact_lines = result | beam.Map(join_info)
-  # [END model_group_by_key_cogroupbykey_tuple]
-  contact_lines | beam.io.WriteToText(output_path)
-  p.run()
+  with TestPipeline() as p:  # Use TestPipeline for testing.
+    # [START model_group_by_key_cogroupbykey_tuple]
+    # Each data set is represented by key-value pairs in separate PCollections.
+    # Both data sets share a common key type (in this example str).
+    # The email_list contains values such as: ('joe', 'joe@example.com') with
+    # multiple possible values for each key.
+    # The phone_list contains values such as: ('mary': '111-222-3333') with
+    # multiple possible values for each key.
+    emails = p | 'email' >> beam.Create(email_list)
+    phones = p | 'phone' >> beam.Create(phone_list)
+    # The result PCollection contains one key-value element for each key in the
+    # input PCollections. The key of the pair will be the key from the input and
+    # the value will be a dictionary with two entries: 'emails' - an iterable of
+    # all values for the current key in the emails PCollection and 'phones': an
+    # iterable of all values for the current key in the phones PCollection.
+    # For instance, if 'emails' contained ('joe', 'joe@example.com') and
+    # ('joe', 'joe@gmail.com'), then 'result' will contain the element
+    # ('joe', {'emails': ['joe@example.com', 'joe@gmail.com'], 'phones': ...})
+    result = {'emails': emails, 'phones': phones} | beam.CoGroupByKey()
+
+    def join_info((name, info)):
+      return '; '.join(['%s' % name,
+                        '%s' % ','.join(info['emails']),
+                        '%s' % ','.join(info['phones'])])
+
+    contact_lines = result | beam.Map(join_info)
+    # [END model_group_by_key_cogroupbykey_tuple]
+    contact_lines | beam.io.WriteToText(output_path)
 
 
 def model_join_using_side_inputs(
@@ -1121,35 +1094,34 @@ def model_join_using_side_inputs(
   import apache_beam as beam
   from apache_beam.pvalue import AsIter
 
-  p = TestPipeline()  # Use TestPipeline for testing.
-  # [START model_join_using_side_inputs]
-  # This code performs a join by receiving the set of names as an input and
-  # passing PCollections that contain emails and phone numbers as side inputs
-  # instead of using CoGroupByKey.
-  names = p | 'names' >> beam.Create(name_list)
-  emails = p | 'email' >> beam.Create(email_list)
-  phones = p | 'phone' >> beam.Create(phone_list)
-
-  def join_info(name, emails, phone_numbers):
-    filtered_emails = []
-    for name_in_list, email in emails:
-      if name_in_list == name:
-        filtered_emails.append(email)
-
-    filtered_phone_numbers = []
-    for name_in_list, phone_number in phone_numbers:
-      if name_in_list == name:
-        filtered_phone_numbers.append(phone_number)
-
-    return '; '.join(['%s' % name,
-                      '%s' % ','.join(filtered_emails),
-                      '%s' % ','.join(filtered_phone_numbers)])
-
-  contact_lines = names | 'CreateContacts' >> beam.core.Map(
-      join_info, AsIter(emails), AsIter(phones))
-  # [END model_join_using_side_inputs]
-  contact_lines | beam.io.WriteToText(output_path)
-  p.run()
+  with TestPipeline() as p:  # Use TestPipeline for testing.
+    # [START model_join_using_side_inputs]
+    # This code performs a join by receiving the set of names as an input and
+    # passing PCollections that contain emails and phone numbers as side inputs
+    # instead of using CoGroupByKey.
+    names = p | 'names' >> beam.Create(name_list)
+    emails = p | 'email' >> beam.Create(email_list)
+    phones = p | 'phone' >> beam.Create(phone_list)
+
+    def join_info(name, emails, phone_numbers):
+      filtered_emails = []
+      for name_in_list, email in emails:
+        if name_in_list == name:
+          filtered_emails.append(email)
+
+      filtered_phone_numbers = []
+      for name_in_list, phone_number in phone_numbers:
+        if name_in_list == name:
+          filtered_phone_numbers.append(phone_number)
+
+      return '; '.join(['%s' % name,
+                        '%s' % ','.join(filtered_emails),
+                        '%s' % ','.join(filtered_phone_numbers)])
+
+    contact_lines = names | 'CreateContacts' >> beam.core.Map(
+        join_info, AsIter(emails), AsIter(phones))
+    # [END model_join_using_side_inputs]
+    contact_lines | beam.io.WriteToText(output_path)
 
 
 # [START model_library_transforms_keys]
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index f7b51a75e7eb..6654fef654cb 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -119,7 +119,6 @@ def capitals(word):
     self.assertEqual({'A', 'C'}, set(all_capitals))
 
   def test_pardo_with_label(self):
-    # pylint: disable=line-too-long
     words = ['aa', 'bbc', 'defg']
     # [START model_pardo_with_label]
     result = words | 'CountUniqueLetters' >> beam.Map(
@@ -129,41 +128,41 @@ def test_pardo_with_label(self):
     self.assertEqual({1, 2, 4}, set(result))
 
   def test_pardo_side_input(self):
-    p = TestPipeline()
-    words = p | 'start' >> beam.Create(['a', 'bb', 'ccc', 'dddd'])
-
-    # [START model_pardo_side_input]
-    # Callable takes additional arguments.
-    def filter_using_length(word, lower_bound, upper_bound=float('inf')):
-      if lower_bound <= len(word) <= upper_bound:
-        yield word
-
-    # Construct a deferred side input.
-    avg_word_len = (words
-                    | beam.Map(len)
-                    | beam.CombineGlobally(beam.combiners.MeanCombineFn()))
-
-    # Call with explicit side inputs.
-    small_words = words | 'small' >> beam.FlatMap(filter_using_length, 0, 3)
-
-    # A single deferred side input.
-    larger_than_average = (words | 'large' >> beam.FlatMap(
-        filter_using_length,
-        lower_bound=pvalue.AsSingleton(avg_word_len)))
-
-    # Mix and match.
-    small_but_nontrivial = words | beam.FlatMap(filter_using_length,
-                                                lower_bound=2,
-                                                upper_bound=pvalue.AsSingleton(
-                                                    avg_word_len))
-    # [END model_pardo_side_input]
-
-    assert_that(small_words, equal_to(['a', 'bb', 'ccc']))
-    assert_that(larger_than_average, equal_to(['ccc', 'dddd']),
-                label='larger_than_average')
-    assert_that(small_but_nontrivial, equal_to(['bb']),
-                label='small_but_not_trivial')
-    p.run()
+    # pylint: disable=line-too-long
+    with TestPipeline() as p:
+      words = p | 'start' >> beam.Create(['a', 'bb', 'ccc', 'dddd'])
+
+      # [START model_pardo_side_input]
+      # Callable takes additional arguments.
+      def filter_using_length(word, lower_bound, upper_bound=float('inf')):
+        if lower_bound <= len(word) <= upper_bound:
+          yield word
+
+      # Construct a deferred side input.
+      avg_word_len = (words
+                      | beam.Map(len)
+                      | beam.CombineGlobally(beam.combiners.MeanCombineFn()))
+
+      # Call with explicit side inputs.
+      small_words = words | 'small' >> beam.FlatMap(filter_using_length, 0, 3)
+
+      # A single deferred side input.
+      larger_than_average = (words | 'large' >> beam.FlatMap(
+          filter_using_length,
+          lower_bound=pvalue.AsSingleton(avg_word_len)))
+
+      # Mix and match.
+      small_but_nontrivial = words | beam.FlatMap(
+          filter_using_length,
+          lower_bound=2,
+          upper_bound=pvalue.AsSingleton(avg_word_len))
+      # [END model_pardo_side_input]
+
+      assert_that(small_words, equal_to(['a', 'bb', 'ccc']))
+      assert_that(larger_than_average, equal_to(['ccc', 'dddd']),
+                  label='larger_than_average')
+      assert_that(small_but_nontrivial, equal_to(['bb']),
+                  label='small_but_not_trivial')
 
   def test_pardo_side_input_dofn(self):
     words = ['a', 'bb', 'ccc', 'dddd']
@@ -307,10 +306,9 @@ def expand(self, pcoll):
 
   def test_runtime_checks_off(self):
     # pylint: disable=expression-not-assigned
-    p = TestPipeline()
-    # [START type_hints_runtime_off]
-    p | beam.Create(['a']) | beam.Map(lambda x: 3).with_output_types(str)
-    p.run()
+    with TestPipeline() as p:
+      # [START type_hints_runtime_off]
+      p | beam.Create(['a']) | beam.Map(lambda x: 3).with_output_types(str)
     # [END type_hints_runtime_off]
 
   def test_runtime_checks_on(self):
@@ -323,47 +321,45 @@ def test_runtime_checks_on(self):
       # [END type_hints_runtime_on]
 
   def test_deterministic_key(self):
-    p = TestPipeline()
-    lines = (p | beam.Create(
-        ['banana,fruit,3', 'kiwi,fruit,2', 'kiwi,fruit,2', 'zucchini,veg,3']))
+    with TestPipeline() as p:
+      lines = (p | beam.Create(
+          ['banana,fruit,3', 'kiwi,fruit,2', 'kiwi,fruit,2', 'zucchini,veg,3']))
 
-    # For pickling
-    global Player  # pylint: disable=global-variable-not-assigned
+      # For pickling
+      global Player  # pylint: disable=global-variable-not-assigned
 
-    # [START type_hints_deterministic_key]
-    class Player(object):
-      def __init__(self, team, name):
-        self.team = team
-        self.name = name
+      # [START type_hints_deterministic_key]
+      class Player(object):
+        def __init__(self, team, name):
+          self.team = team
+          self.name = name
 
-    class PlayerCoder(beam.coders.Coder):
-      def encode(self, player):
-        return '%s:%s' % (player.team, player.name)
+      class PlayerCoder(beam.coders.Coder):
+        def encode(self, player):
+          return '%s:%s' % (player.team, player.name)
 
-      def decode(self, s):
-        return Player(*s.split(':'))
+        def decode(self, s):
+          return Player(*s.split(':'))
 
-      def is_deterministic(self):
-        return True
+        def is_deterministic(self):
+          return True
 
-    beam.coders.registry.register_coder(Player, PlayerCoder)
+      beam.coders.registry.register_coder(Player, PlayerCoder)
 
-    def parse_player_and_score(csv):
-      name, team, score = csv.split(',')
-      return Player(team, name), int(score)
+      def parse_player_and_score(csv):
+        name, team, score = csv.split(',')
+        return Player(team, name), int(score)
 
-    totals = (
-        lines
-        | beam.Map(parse_player_and_score)
-        | beam.CombinePerKey(sum).with_input_types(
-            beam.typehints.Tuple[Player, int]))
-    # [END type_hints_deterministic_key]
+      totals = (
+          lines
+          | beam.Map(parse_player_and_score)
+          | beam.CombinePerKey(sum).with_input_types(
+              beam.typehints.Tuple[Player, int]))
+      # [END type_hints_deterministic_key]
 
-    assert_that(
-        totals | beam.Map(lambda (k, v): (k.name, v)),
-        equal_to([('banana', 3), ('kiwi', 4), ('zucchini', 3)]))
-
-    p.run()
+      assert_that(
+          totals | beam.Map(lambda (k, v): (k.name, v)),
+          equal_to([('banana', 3), ('kiwi', 4), ('zucchini', 3)]))
 
 
 class SnippetsTest(unittest.TestCase):
@@ -802,109 +798,104 @@ def test_count(self):
     self.assertEqual({('cat', 3), ('dog', 2)}, set(perkey_counts))
 
   def test_setting_fixed_windows(self):
-    p = TestPipeline()
-    unkeyed_items = p | beam.Create([22, 33, 55, 100, 115, 120])
-    items = (unkeyed_items
-             | 'key' >> beam.Map(
-                 lambda x: beam.window.TimestampedValue(('k', x), x)))
-    # [START setting_fixed_windows]
-    from apache_beam import window
-    fixed_windowed_items = (
-        items | 'window' >> beam.WindowInto(window.FixedWindows(60)))
-    # [END setting_fixed_windows]
-    summed = (fixed_windowed_items
-              | 'group' >> beam.GroupByKey()
-              | 'combine' >> beam.CombineValues(sum))
-    unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    assert_that(unkeyed, equal_to([110, 215, 120]))
-    p.run()
+    with TestPipeline() as p:
+      unkeyed_items = p | beam.Create([22, 33, 55, 100, 115, 120])
+      items = (unkeyed_items
+               | 'key' >> beam.Map(
+                   lambda x: beam.window.TimestampedValue(('k', x), x)))
+      # [START setting_fixed_windows]
+      from apache_beam import window
+      fixed_windowed_items = (
+          items | 'window' >> beam.WindowInto(window.FixedWindows(60)))
+      # [END setting_fixed_windows]
+      summed = (fixed_windowed_items
+                | 'group' >> beam.GroupByKey()
+                | 'combine' >> beam.CombineValues(sum))
+      unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+      assert_that(unkeyed, equal_to([110, 215, 120]))
 
   def test_setting_sliding_windows(self):
-    p = TestPipeline()
-    unkeyed_items = p | beam.Create([2, 16, 23])
-    items = (unkeyed_items
-             | 'key' >> beam.Map(
-                 lambda x: beam.window.TimestampedValue(('k', x), x)))
-    # [START setting_sliding_windows]
-    from apache_beam import window
-    sliding_windowed_items = (
-        items | 'window' >> beam.WindowInto(window.SlidingWindows(30, 5)))
-    # [END setting_sliding_windows]
-    summed = (sliding_windowed_items
-              | 'group' >> beam.GroupByKey()
-              | 'combine' >> beam.CombineValues(sum))
-    unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    assert_that(unkeyed,
-                equal_to([2, 2, 2, 18, 23, 39, 39, 39, 41, 41]))
-    p.run()
+    with TestPipeline() as p:
+      unkeyed_items = p | beam.Create([2, 16, 23])
+      items = (unkeyed_items
+               | 'key' >> beam.Map(
+                   lambda x: beam.window.TimestampedValue(('k', x), x)))
+      # [START setting_sliding_windows]
+      from apache_beam import window
+      sliding_windowed_items = (
+          items | 'window' >> beam.WindowInto(window.SlidingWindows(30, 5)))
+      # [END setting_sliding_windows]
+      summed = (sliding_windowed_items
+                | 'group' >> beam.GroupByKey()
+                | 'combine' >> beam.CombineValues(sum))
+      unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+      assert_that(unkeyed,
+                  equal_to([2, 2, 2, 18, 23, 39, 39, 39, 41, 41]))
 
   def test_setting_session_windows(self):
-    p = TestPipeline()
-    unkeyed_items = p | beam.Create([2, 11, 16, 27])
-    items = (unkeyed_items
-             | 'key' >> beam.Map(
-                 lambda x: beam.window.TimestampedValue(('k', x), x)))
-    # [START setting_session_windows]
-    from apache_beam import window
-    session_windowed_items = (
-        items | 'window' >> beam.WindowInto(window.Sessions(10)))
-    # [END setting_session_windows]
-    summed = (session_windowed_items
-              | 'group' >> beam.GroupByKey()
-              | 'combine' >> beam.CombineValues(sum))
-    unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    assert_that(unkeyed,
-                equal_to([29, 27]))
-    p.run()
+    with TestPipeline() as p:
+      unkeyed_items = p | beam.Create([2, 11, 16, 27])
+      items = (unkeyed_items
+               | 'key' >> beam.Map(
+                   lambda x: beam.window.TimestampedValue(('k', x), x)))
+      # [START setting_session_windows]
+      from apache_beam import window
+      session_windowed_items = (
+          items | 'window' >> beam.WindowInto(window.Sessions(10)))
+      # [END setting_session_windows]
+      summed = (session_windowed_items
+                | 'group' >> beam.GroupByKey()
+                | 'combine' >> beam.CombineValues(sum))
+      unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+      assert_that(unkeyed,
+                  equal_to([29, 27]))
 
   def test_setting_global_window(self):
-    p = TestPipeline()
-    unkeyed_items = p | beam.Create([2, 11, 16, 27])
-    items = (unkeyed_items
-             | 'key' >> beam.Map(
-                 lambda x: beam.window.TimestampedValue(('k', x), x)))
-    # [START setting_global_window]
-    from apache_beam import window
-    session_windowed_items = (
-        items | 'window' >> beam.WindowInto(window.GlobalWindows()))
-    # [END setting_global_window]
-    summed = (session_windowed_items
-              | 'group' >> beam.GroupByKey()
-              | 'combine' >> beam.CombineValues(sum))
-    unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    assert_that(unkeyed, equal_to([56]))
-    p.run()
+    with TestPipeline() as p:
+      unkeyed_items = p | beam.Create([2, 11, 16, 27])
+      items = (unkeyed_items
+               | 'key' >> beam.Map(
+                   lambda x: beam.window.TimestampedValue(('k', x), x)))
+      # [START setting_global_window]
+      from apache_beam import window
+      session_windowed_items = (
+          items | 'window' >> beam.WindowInto(window.GlobalWindows()))
+      # [END setting_global_window]
+      summed = (session_windowed_items
+                | 'group' >> beam.GroupByKey()
+                | 'combine' >> beam.CombineValues(sum))
+      unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+      assert_that(unkeyed, equal_to([56]))
 
   def test_setting_timestamp(self):
-    p = TestPipeline()
-    unkeyed_items = p | beam.Create([12, 30, 60, 61, 66])
-    items = (unkeyed_items | 'key' >> beam.Map(lambda x: ('k', x)))
+    with TestPipeline() as p:
+      unkeyed_items = p | beam.Create([12, 30, 60, 61, 66])
+      items = (unkeyed_items | 'key' >> beam.Map(lambda x: ('k', x)))
 
-    def extract_timestamp_from_log_entry(entry):
-      return entry[1]
+      def extract_timestamp_from_log_entry(entry):
+        return entry[1]
 
-    # [START setting_timestamp]
-    class AddTimestampDoFn(beam.DoFn):
+      # [START setting_timestamp]
+      class AddTimestampDoFn(beam.DoFn):
 
-      def process(self, element):
-        # Extract the numeric Unix seconds-since-epoch timestamp to be
-        # associated with the current log entry.
-        unix_timestamp = extract_timestamp_from_log_entry(element)
-        # Wrap and emit the current entry and new timestamp in a
-        # TimestampedValue.
-        yield beam.window.TimestampedValue(element, unix_timestamp)
-
-    timestamped_items = items | 'timestamp' >> beam.ParDo(AddTimestampDoFn())
-    # [END setting_timestamp]
-    fixed_windowed_items = (
-        timestamped_items | 'window' >> beam.WindowInto(
-            beam.window.FixedWindows(60)))
-    summed = (fixed_windowed_items
-              | 'group' >> beam.GroupByKey()
-              | 'combine' >> beam.CombineValues(sum))
-    unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
-    assert_that(unkeyed, equal_to([42, 187]))
-    p.run()
+        def process(self, element):
+          # Extract the numeric Unix seconds-since-epoch timestamp to be
+          # associated with the current log entry.
+          unix_timestamp = extract_timestamp_from_log_entry(element)
+          # Wrap and emit the current entry and new timestamp in a
+          # TimestampedValue.
+          yield beam.window.TimestampedValue(element, unix_timestamp)
+
+      timestamped_items = items | 'timestamp' >> beam.ParDo(AddTimestampDoFn())
+      # [END setting_timestamp]
+      fixed_windowed_items = (
+          timestamped_items | 'window' >> beam.WindowInto(
+              beam.window.FixedWindows(60)))
+      summed = (fixed_windowed_items
+                | 'group' >> beam.GroupByKey()
+                | 'combine' >> beam.CombineValues(sum))
+      unkeyed = summed | 'unkey' >> beam.Map(lambda x: x[1])
+      assert_that(unkeyed, equal_to([42, 187]))
 
 
 class PTransformTest(unittest.TestCase):
@@ -919,10 +910,9 @@ def expand(self, pcoll):
         return pcoll | beam.Map(lambda x: len(x))
     # [END model_composite_transform]
 
-    p = TestPipeline()
-    lengths = p | beam.Create(["a", "ab", "abc"]) | ComputeWordLengths()
-    assert_that(lengths, equal_to([1, 2, 3]))
-    p.run()
+    with TestPipeline() as p:
+      lengths = p | beam.Create(["a", "ab", "abc"]) | ComputeWordLengths()
+      assert_that(lengths, equal_to([1, 2, 3]))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/streaming_wordcap.py b/sdks/python/apache_beam/examples/streaming_wordcap.py
index d0cc8a201b41..ce43e1f09849 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcap.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcap.py
@@ -41,22 +41,20 @@ def run(argv=None):
       help='Output PubSub topic of the form "/topics/<PROJECT>/<TOPIC>".')
   known_args, pipeline_args = parser.parse_known_args(argv)
 
-  p = beam.Pipeline(argv=pipeline_args)
+  with beam.Pipeline(argv=pipeline_args) as p:
 
-  # Read the text file[pattern] into a PCollection.
-  lines = p | beam.io.Read(
-      beam.io.PubSubSource(known_args.input_topic))
+    # Read the text file[pattern] into a PCollection.
+    lines = p | beam.io.Read(
+        beam.io.PubSubSource(known_args.input_topic))
 
-  # Capitalize the characters in each line.
-  transformed = (lines
-                 | 'capitalize' >> (beam.Map(lambda x: x.upper())))
+    # Capitalize the characters in each line.
+    transformed = (lines
+                   | 'capitalize' >> (beam.Map(lambda x: x.upper())))
 
-  # Write to PubSub.
-  # pylint: disable=expression-not-assigned
-  transformed | beam.io.Write(
-      beam.io.PubSubSink(known_args.output_topic))
-
-  p.run().wait_until_finish()
+    # Write to PubSub.
+    # pylint: disable=expression-not-assigned
+    transformed | beam.io.Write(
+        beam.io.PubSubSink(known_args.output_topic))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py
index 4b6aecc98c4a..e9d5dbefa835 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcount.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcount.py
@@ -44,29 +44,27 @@ def run(argv=None):
       help='Output PubSub topic of the form "/topics/<PROJECT>/<TOPIC>".')
   known_args, pipeline_args = parser.parse_known_args(argv)
 
-  p = beam.Pipeline(argv=pipeline_args)
-
-  # Read the text file[pattern] into a PCollection.
-  lines = p | 'read' >> beam.io.Read(
-      beam.io.PubSubSource(known_args.input_topic))
-
-  # Capitalize the characters in each line.
-  transformed = (lines
-                 | 'Split' >> (
-                     beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
-                     .with_output_types(unicode))
-                 | 'PairWithOne' >> beam.Map(lambda x: (x, 1))
-                 | beam.WindowInto(window.FixedWindows(15, 0))
-                 | 'Group' >> beam.GroupByKey()
-                 | 'Count' >> beam.Map(lambda (word, ones): (word, sum(ones)))
-                 | 'Format' >> beam.Map(lambda tup: '%s: %d' % tup))
-
-  # Write to PubSub.
-  # pylint: disable=expression-not-assigned
-  transformed | 'pubsub_write' >> beam.io.Write(
-      beam.io.PubSubSink(known_args.output_topic))
-
-  p.run().wait_until_finish()
+  with beam.Pipeline(argv=pipeline_args) as p:
+
+    # Read the text file[pattern] into a PCollection.
+    lines = p | 'read' >> beam.io.Read(
+        beam.io.PubSubSource(known_args.input_topic))
+
+    # Capitalize the characters in each line.
+    transformed = (lines
+                   | 'Split' >> (
+                       beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+                       .with_output_types(unicode))
+                   | 'PairWithOne' >> beam.Map(lambda x: (x, 1))
+                   | beam.WindowInto(window.FixedWindows(15, 0))
+                   | 'Group' >> beam.GroupByKey()
+                   | 'Count' >> beam.Map(lambda (word, ones): (word, sum(ones)))
+                   | 'Format' >> beam.Map(lambda tup: '%s: %d' % tup))
+
+    # Write to PubSub.
+    # pylint: disable=expression-not-assigned
+    transformed | 'pubsub_write' >> beam.io.Write(
+        beam.io.PubSubSink(known_args.output_topic))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py
index e7e542a74219..34dedb2b819a 100644
--- a/sdks/python/apache_beam/examples/wordcount.py
+++ b/sdks/python/apache_beam/examples/wordcount.py
@@ -102,7 +102,6 @@ def run(argv=None):
   # pylint: disable=expression-not-assigned
   output | 'write' >> WriteToText(known_args.output)
 
-  # Actually run the pipeline (all operations above are deferred).
   result = p.run()
   result.wait_until_finish()
 
diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py
index ca9f7b6371de..c0ffd356364c 100644
--- a/sdks/python/apache_beam/examples/wordcount_debugging.py
+++ b/sdks/python/apache_beam/examples/wordcount_debugging.py
@@ -118,35 +118,32 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
-
-  # Read the text file[pattern] into a PCollection, count the occurrences of
-  # each word and filter by a list of words.
-  filtered_words = (
-      p | 'read' >> ReadFromText(known_args.input)
-      | CountWords()
-      | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
-
-  # assert_that is a convenient PTransform that checks a PCollection has an
-  # expected value. Asserts are best used in unit tests with small data sets but
-  # is demonstrated here as a teaching tool.
-  #
-  # Note assert_that does not provide any output and that successful completion
-  # of the Pipeline implies that the expectations were  met. Learn more at
-  # https://cloud.google.com/dataflow/pipelines/testing-your-pipeline on how to
-  # test your pipeline.
-  assert_that(
-      filtered_words, equal_to([('Flourish', 3), ('stomach', 1)]))
-
-  # Format the counts into a PCollection of strings and write the output using a
-  # "Write" transform that has side effects.
-  # pylint: disable=unused-variable
-  output = (filtered_words
-            | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
-            | 'write' >> WriteToText(known_args.output))
-
-  # Actually run the pipeline (all operations above are deferred).
-  p.run().wait_until_finish()
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    # Read the text file[pattern] into a PCollection, count the occurrences of
+    # each word and filter by a list of words.
+    filtered_words = (
+        p | 'read' >> ReadFromText(known_args.input)
+        | CountWords()
+        | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))
+
+    # assert_that is a convenient PTransform that checks a PCollection has an
+    # expected value. Asserts are best used in unit tests with small data sets
+    # but is demonstrated here as a teaching tool.
+    #
+    # Note assert_that does not provide any output and that successful
+    # completion of the Pipeline implies that the expectations were  met. Learn
+    # more at https://cloud.google.com/dataflow/pipelines/testing-your-pipeline
+    # on how to best test your pipeline.
+    assert_that(
+        filtered_words, equal_to([('Flourish', 3), ('stomach', 1)]))
+
+    # Format the counts into a PCollection of strings and write the output using
+    # a "Write" transform that has side effects.
+    # pylint: disable=unused-variable
+    output = (filtered_words
+              | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
+              | 'write' >> WriteToText(known_args.output))
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/wordcount_minimal.py b/sdks/python/apache_beam/examples/wordcount_minimal.py
index 5109c089f993..76b0a221df7a 100644
--- a/sdks/python/apache_beam/examples/wordcount_minimal.py
+++ b/sdks/python/apache_beam/examples/wordcount_minimal.py
@@ -92,28 +92,25 @@ def run(argv=None):
   # workflow rely on global context (e.g., a module imported at module level).
   pipeline_options = PipelineOptions(pipeline_args)
   pipeline_options.view_as(SetupOptions).save_main_session = True
-  p = beam.Pipeline(options=pipeline_options)
+  with beam.Pipeline(options=pipeline_options) as p:
 
-  # Read the text file[pattern] into a PCollection.
-  lines = p | 'read' >> ReadFromText(known_args.input)
+    # Read the text file[pattern] into a PCollection.
+    lines = p | ReadFromText(known_args.input)
 
-  # Count the occurrences of each word.
-  counts = (lines
-            | 'split' >> (beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
-                          .with_output_types(unicode))
-            | 'pair_with_one' >> beam.Map(lambda x: (x, 1))
-            | 'group' >> beam.GroupByKey()
-            | 'count' >> beam.Map(lambda (word, ones): (word, sum(ones))))
+    # Count the occurrences of each word.
+    counts = (
+        lines
+        | 'Split' >> (beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+                      .with_output_types(unicode))
+        | 'PairWithOne' >> beam.Map(lambda x: (x, 1))
+        | 'GroupAndSum' >> beam.CombinePerKey(sum))
 
-  # Format the counts into a PCollection of strings.
-  output = counts | 'format' >> beam.Map(lambda (word, c): '%s: %s' % (word, c))
+    # Format the counts into a PCollection of strings.
+    output = counts | 'Format' >> beam.Map(lambda (w, c): '%s: %s' % (w, c))
 
-  # Write the output using a "Write" transform that has side effects.
-  # pylint: disable=expression-not-assigned
-  output | 'write' >> WriteToText(known_args.output)
-
-  # Actually run the pipeline (all operations above are deferred).
-  p.run().wait_until_finish()
+    # Write the output using a "Write" transform that has side effects.
+    # pylint: disable=expression-not-assigned
+    output | WriteToText(known_args.output)
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/io/filebasedsink_test.py b/sdks/python/apache_beam/io/filebasedsink_test.py
index 1f6aeee56677..7c8ddb4072ef 100644
--- a/sdks/python/apache_beam/io/filebasedsink_test.py
+++ b/sdks/python/apache_beam/io/filebasedsink_test.py
@@ -146,9 +146,8 @@ def test_empty_write(self):
     sink = MyFileBasedSink(
         temp_path, file_name_suffix='.output', coder=coders.ToStringCoder()
     )
-    p = TestPipeline()
-    p | beam.Create([]) | beam.io.Write(sink)  # pylint: disable=expression-not-assigned
-    p.run()
+    with TestPipeline() as p:
+      p | beam.Create([]) | beam.io.Write(sink)  # pylint: disable=expression-not-assigned
     self.assertEqual(
         open(temp_path + '-00000-of-00001.output').read(), '[start][end]')
 
@@ -160,9 +159,8 @@ def test_static_value_provider_empty_write(self):
         file_name_suffix=StaticValueProvider(value_type=str, value='.output'),
         coder=coders.ToStringCoder()
     )
-    p = TestPipeline()
-    p | beam.Create([]) | beam.io.Write(sink)  # pylint: disable=expression-not-assigned
-    p.run()
+    with TestPipeline() as p:
+      p | beam.Create([]) | beam.io.Write(sink)  # pylint: disable=expression-not-assigned
     self.assertEqual(
         open(temp_path.get() + '-00000-of-00001.output').read(), '[start][end]')
 
@@ -174,10 +172,8 @@ def test_fixed_shard_write(self):
         num_shards=3,
         shard_name_template='_NN_SSS_',
         coder=coders.ToStringCoder())
-    p = TestPipeline()
-    p | beam.Create(['a', 'b']) | beam.io.Write(sink)  # pylint: disable=expression-not-assigned
-
-    p.run()
+    with TestPipeline() as p:
+      p | beam.Create(['a', 'b']) | beam.io.Write(sink)  # pylint: disable=expression-not-assigned
 
     concat = ''.join(
         open(temp_path + '_03_%03d_.output' % shard_num).read()
diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index 5048534b3852..9093abfccfc3 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -28,19 +28,18 @@
 Typical usage:
 
   # Create a pipeline object using a local runner for execution.
-  p = beam.Pipeline('DirectRunner')
+  with beam.Pipeline('DirectRunner') as p:
 
-  # Add to the pipeline a "Create" transform. When executed this
-  # transform will produce a PCollection object with the specified values.
-  pcoll = p | 'Create' >> beam.Create([1, 2, 3])
+    # Add to the pipeline a "Create" transform. When executed this
+    # transform will produce a PCollection object with the specified values.
+    pcoll = p | 'Create' >> beam.Create([1, 2, 3])
 
-  # Another transform could be applied to pcoll, e.g., writing to a text file.
-  # For other transforms, refer to transforms/ directory.
-  pcoll | 'Write' >> beam.io.WriteToText('./output')
+    # Another transform could be applied to pcoll, e.g., writing to a text file.
+    # For other transforms, refer to transforms/ directory.
+    pcoll | 'Write' >> beam.io.WriteToText('./output')
 
-  # run() will execute the DAG stored in the pipeline.  The execution of the
-  # nodes visited is done using the specified local runner.
-  p.run()
+    # run() will execute the DAG stored in the pipeline.  The execution of the
+    # nodes visited is done using the specified local runner.
 
 """
 
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
index 946a60a3675c..c79fec864acb 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -247,26 +247,23 @@ def match(actual):
     pipeline.run()
 
   def test_tuple_combine_fn(self):
-    p = TestPipeline()
-    result = (
-        p
-        | Create([('a', 100, 0.0), ('b', 10, -1), ('c', 1, 100)])
-        | beam.CombineGlobally(combine.TupleCombineFn(max,
-                                                      combine.MeanCombineFn(),
-                                                      sum)).without_defaults())
-    assert_that(result, equal_to([('c', 111.0 / 3, 99.0)]))
-    p.run()
+    with TestPipeline() as p:
+      result = (
+          p
+          | Create([('a', 100, 0.0), ('b', 10, -1), ('c', 1, 100)])
+          | beam.CombineGlobally(combine.TupleCombineFn(
+              max, combine.MeanCombineFn(), sum)).without_defaults())
+      assert_that(result, equal_to([('c', 111.0 / 3, 99.0)]))
 
   def test_tuple_combine_fn_without_defaults(self):
-    p = TestPipeline()
-    result = (
-        p
-        | Create([1, 1, 2, 3])
-        | beam.CombineGlobally(
-            combine.TupleCombineFn(min, combine.MeanCombineFn(), max)
-            .with_common_input()).without_defaults())
-    assert_that(result, equal_to([(1, 7.0 / 4, 3)]))
-    p.run()
+    with TestPipeline() as p:
+      result = (
+          p
+          | Create([1, 1, 2, 3])
+          | beam.CombineGlobally(
+              combine.TupleCombineFn(min, combine.MeanCombineFn(), max)
+              .with_common_input()).without_defaults())
+      assert_that(result, equal_to([(1, 7.0 / 4, 3)]))
 
   def test_to_list_and_to_dict(self):
     pipeline = TestPipeline()
@@ -295,29 +292,26 @@ def match(actual):
     pipeline.run()
 
   def test_combine_globally_with_default(self):
-    p = TestPipeline()
-    assert_that(p | Create([]) | CombineGlobally(sum), equal_to([0]))
-    p.run()
+    with TestPipeline() as p:
+      assert_that(p | Create([]) | CombineGlobally(sum), equal_to([0]))
 
   def test_combine_globally_without_default(self):
-    p = TestPipeline()
-    result = p | Create([]) | CombineGlobally(sum).without_defaults()
-    assert_that(result, equal_to([]))
-    p.run()
+    with TestPipeline() as p:
+      result = p | Create([]) | CombineGlobally(sum).without_defaults()
+      assert_that(result, equal_to([]))
 
   def test_combine_globally_with_default_side_input(self):
-    class CombineWithSideInput(PTransform):
+    class SideInputCombine(PTransform):
       def expand(self, pcoll):
         side = pcoll | CombineGlobally(sum).as_singleton_view()
         main = pcoll.pipeline | Create([None])
         return main | Map(lambda _, s: s, side)
 
-    p = TestPipeline()
-    result1 = p | 'i1' >> Create([]) | 'c1' >> CombineWithSideInput()
-    result2 = p | 'i2' >> Create([1, 2, 3, 4]) | 'c2' >> CombineWithSideInput()
-    assert_that(result1, equal_to([0]), label='r1')
-    assert_that(result2, equal_to([10]), label='r2')
-    p.run()
+    with TestPipeline() as p:
+      result1 = p | 'i1' >> Create([]) | 'c1' >> SideInputCombine()
+      result2 = p | 'i2' >> Create([1, 2, 3, 4]) | 'c2' >> SideInputCombine()
+      assert_that(result1, equal_to([0]), label='r1')
+      assert_that(result2, equal_to([10]), label='r2')
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py
index fd1bb9d5250b..977a364ad883 100644
--- a/sdks/python/apache_beam/transforms/window_test.py
+++ b/sdks/python/apache_beam/transforms/window_test.py
@@ -167,90 +167,85 @@ def timestamped_key_values(self, pipeline, key, *timestamps):
             | Map(lambda x: WindowedValue((key, x), x, [GlobalWindow()])))
 
   def test_sliding_windows(self):
-    p = TestPipeline()
-    pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3)
-    result = (pcoll
-              | 'w' >> WindowInto(SlidingWindows(period=2, size=4))
-              | GroupByKey()
-              | reify_windows)
-    expected = [('key @ [-2.0, 2.0)', [1]),
-                ('key @ [0.0, 4.0)', [1, 2, 3]),
-                ('key @ [2.0, 6.0)', [2, 3])]
-    assert_that(result, equal_to(expected))
-    p.run()
+    with TestPipeline() as p:
+      pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3)
+      result = (pcoll
+                | 'w' >> WindowInto(SlidingWindows(period=2, size=4))
+                | GroupByKey()
+                | reify_windows)
+      expected = [('key @ [-2.0, 2.0)', [1]),
+                  ('key @ [0.0, 4.0)', [1, 2, 3]),
+                  ('key @ [2.0, 6.0)', [2, 3])]
+      assert_that(result, equal_to(expected))
 
   def test_sessions(self):
-    p = TestPipeline()
-    pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3, 20, 35, 27)
-    result = (pcoll
-              | 'w' >> WindowInto(Sessions(10))
-              | GroupByKey()
-              | sort_values
-              | reify_windows)
-    expected = [('key @ [1.0, 13.0)', [1, 2, 3]),
-                ('key @ [20.0, 45.0)', [20, 27, 35])]
-    assert_that(result, equal_to(expected))
-    p.run()
+    with TestPipeline() as p:
+      pcoll = self.timestamped_key_values(p, 'key', 1, 2, 3, 20, 35, 27)
+      result = (pcoll
+                | 'w' >> WindowInto(Sessions(10))
+                | GroupByKey()
+                | sort_values
+                | reify_windows)
+      expected = [('key @ [1.0, 13.0)', [1, 2, 3]),
+                  ('key @ [20.0, 45.0)', [20, 27, 35])]
+      assert_that(result, equal_to(expected))
 
   def test_timestamped_value(self):
-    p = TestPipeline()
-    result = (p
-              | 'start' >> Create([(k, k) for k in range(10)])
-              | Map(lambda (x, t): TimestampedValue(x, t))
-              | 'w' >> WindowInto(FixedWindows(5))
-              | Map(lambda v: ('key', v))
-              | GroupByKey())
-    assert_that(result, equal_to([('key', [0, 1, 2, 3, 4]),
-                                  ('key', [5, 6, 7, 8, 9])]))
-    p.run()
+    with TestPipeline() as p:
+      result = (p
+                | 'start' >> Create([(k, k) for k in range(10)])
+                | Map(lambda (x, t): TimestampedValue(x, t))
+                | 'w' >> WindowInto(FixedWindows(5))
+                | Map(lambda v: ('key', v))
+                | GroupByKey())
+      assert_that(result, equal_to([('key', [0, 1, 2, 3, 4]),
+                                    ('key', [5, 6, 7, 8, 9])]))
 
   def test_rewindow(self):
-    p = TestPipeline()
-    result = (p
-              | Create([(k, k) for k in range(10)])
-              | Map(lambda (x, t): TimestampedValue(x, t))
-              | 'window' >> WindowInto(SlidingWindows(period=2, size=6))
-              # Per the model, each element is now duplicated across
-              # three windows. Rewindowing must preserve this duplication.
-              | 'rewindow' >> WindowInto(FixedWindows(5))
-              | 'rewindow2' >> WindowInto(FixedWindows(5))
-              | Map(lambda v: ('key', v))
-              | GroupByKey())
-    assert_that(result, equal_to([('key', sorted([0, 1, 2, 3, 4] * 3)),
-                                  ('key', sorted([5, 6, 7, 8, 9] * 3))]))
-    p.run()
+    with TestPipeline() as p:
+      result = (p
+                | Create([(k, k) for k in range(10)])
+                | Map(lambda (x, t): TimestampedValue(x, t))
+                | 'window' >> WindowInto(SlidingWindows(period=2, size=6))
+                # Per the model, each element is now duplicated across
+                # three windows. Rewindowing must preserve this duplication.
+                | 'rewindow' >> WindowInto(FixedWindows(5))
+                | 'rewindow2' >> WindowInto(FixedWindows(5))
+                | Map(lambda v: ('key', v))
+                | GroupByKey())
+      assert_that(result, equal_to([('key', sorted([0, 1, 2, 3, 4] * 3)),
+                                    ('key', sorted([5, 6, 7, 8, 9] * 3))]))
 
   def test_timestamped_with_combiners(self):
-    p = TestPipeline()
-    result = (p
-              # Create some initial test values.
-              | 'start' >> Create([(k, k) for k in range(10)])
-              # The purpose of the WindowInto transform is to establish a
-              # FixedWindows windowing function for the PCollection.
-              # It does not bucket elements into windows since the timestamps
-              # from Create are not spaced 5 ms apart and very likely they all
-              # fall into the same window.
-              | 'w' >> WindowInto(FixedWindows(5))
-              # Generate timestamped values using the values as timestamps.
-              # Now there are values 5 ms apart and since Map propagates the
-              # windowing function from input to output the output PCollection
-              # will have elements falling into different 5ms windows.
-              | Map(lambda (x, t): TimestampedValue(x, t))
-              # We add a 'key' to each value representing the index of the
-              # window. This is important since there is no guarantee of
-              # order for the elements of a PCollection.
-              | Map(lambda v: (v / 5, v)))
-    # Sum all elements associated with a key and window. Although it
-    # is called CombinePerKey it is really CombinePerKeyAndWindow the
-    # same way GroupByKey is really GroupByKeyAndWindow.
-    sum_per_window = result | CombinePerKey(sum)
-    # Compute mean per key and window.
-    mean_per_window = result | combiners.Mean.PerKey()
-    assert_that(sum_per_window, equal_to([(0, 10), (1, 35)]),
-                label='assert:sum')
-    assert_that(mean_per_window, equal_to([(0, 2.0), (1, 7.0)]),
-                label='assert:mean')
-    p.run()
+    with TestPipeline() as p:
+      result = (p
+                # Create some initial test values.
+                | 'start' >> Create([(k, k) for k in range(10)])
+                # The purpose of the WindowInto transform is to establish a
+                # FixedWindows windowing function for the PCollection.
+                # It does not bucket elements into windows since the timestamps
+                # from Create are not spaced 5 ms apart and very likely they all
+                # fall into the same window.
+                | 'w' >> WindowInto(FixedWindows(5))
+                # Generate timestamped values using the values as timestamps.
+                # Now there are values 5 ms apart and since Map propagates the
+                # windowing function from input to output the output PCollection
+                # will have elements falling into different 5ms windows.
+                | Map(lambda (x, t): TimestampedValue(x, t))
+                # We add a 'key' to each value representing the index of the
+                # window. This is important since there is no guarantee of
+                # order for the elements of a PCollection.
+                | Map(lambda v: (v / 5, v)))
+      # Sum all elements associated with a key and window. Although it
+      # is called CombinePerKey it is really CombinePerKeyAndWindow the
+      # same way GroupByKey is really GroupByKeyAndWindow.
+      sum_per_window = result | CombinePerKey(sum)
+      # Compute mean per key and window.
+      mean_per_window = result | combiners.Mean.PerKey()
+      assert_that(sum_per_window, equal_to([(0, 10), (1, 35)]),
+                  label='assert:sum')
+      assert_that(mean_per_window, equal_to([(0, 2.0), (1, 7.0)]),
+                  label='assert:mean')
 
 
 class RunnerApiTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/transforms/write_ptransform_test.py b/sdks/python/apache_beam/transforms/write_ptransform_test.py
index e31b9cc9830d..50f0debb0a70 100644
--- a/sdks/python/apache_beam/transforms/write_ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/write_ptransform_test.py
@@ -98,11 +98,10 @@ def _run_write_test(self,
                       return_write_results=True):
     write_to_test_sink = WriteToTestSink(return_init_result,
                                          return_write_results)
-    p = TestPipeline()
-    result = p | beam.Create(data) | write_to_test_sink | beam.Map(list)
+    with TestPipeline() as p:
+      result = p | beam.Create(data) | write_to_test_sink | beam.Map(list)
 
-    assert_that(result, is_empty())
-    p.run()
+      assert_that(result, is_empty())
 
     sink = write_to_test_sink.last_sink
     self.assertIsNotNone(sink)
diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
index 589dc0e5ac82..c81ef320e027 100644
--- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
+++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
@@ -168,12 +168,11 @@ def test_deferred_side_inputs(self):
     @typehints.with_input_types(str, int)
     def repeat(s, times):
       return s * times
-    p = TestPipeline()
-    main_input = p | beam.Create(['a', 'bb', 'c'])
-    side_input = p | 'side' >> beam.Create([3])
-    result = main_input | beam.Map(repeat, pvalue.AsSingleton(side_input))
-    assert_that(result, equal_to(['aaa', 'bbbbbb', 'ccc']))
-    p.run()
+    with TestPipeline() as p:
+      main_input = p | beam.Create(['a', 'bb', 'c'])
+      side_input = p | 'side' >> beam.Create([3])
+      result = main_input | beam.Map(repeat, pvalue.AsSingleton(side_input))
+      assert_that(result, equal_to(['aaa', 'bbbbbb', 'ccc']))
 
     bad_side_input = p | 'bad_side' >> beam.Create(['z'])
     with self.assertRaises(typehints.TypeCheckError):
@@ -183,12 +182,11 @@ def test_deferred_side_input_iterable(self):
     @typehints.with_input_types(str, typehints.Iterable[str])
     def concat(glue, items):
       return glue.join(sorted(items))
-    p = TestPipeline()
-    main_input = p | beam.Create(['a', 'bb', 'c'])
-    side_input = p | 'side' >> beam.Create(['x', 'y', 'z'])
-    result = main_input | beam.Map(concat, pvalue.AsIter(side_input))
-    assert_that(result, equal_to(['xayaz', 'xbbybbz', 'xcycz']))
-    p.run()
+    with TestPipeline() as p:
+      main_input = p | beam.Create(['a', 'bb', 'c'])
+      side_input = p | 'side' >> beam.Create(['x', 'y', 'z'])
+      result = main_input | beam.Map(concat, pvalue.AsIter(side_input))
+      assert_that(result, equal_to(['xayaz', 'xbbybbz', 'xcycz']))
 
     bad_side_input = p | 'bad_side' >> beam.Create([1, 2, 3])
     with self.assertRaises(typehints.TypeCheckError):

From 98a75551064c742d108d8c5ec8fc0783db7761d2 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 15:28:44 -0700
Subject: [PATCH 288/387] Move StepContext to top level

---
 .../translation/utils/NoOpStepContext.java    |  6 +-
 .../runners/core/BaseExecutionContext.java    |  8 +--
 .../apache/beam/runners/core/DoFnRunners.java |  1 -
 .../beam/runners/core/ExecutionContext.java   | 47 -------------
 .../beam/runners/core/SimpleDoFnRunner.java   |  1 -
 .../apache/beam/runners/core/StepContext.java | 70 +++++++++++++++++++
 .../functions/FlinkNoOpStepContext.java       |  2 +-
 .../wrappers/streaming/DoFnOperator.java      |  7 +-
 .../translation/SparkProcessContext.java      |  2 +-
 .../beam/fn/harness/fake/FakeStepContext.java |  2 +-
 10 files changed, 83 insertions(+), 63 deletions(-)
 create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
index 721eecd3d626..241a9856736d 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
@@ -19,8 +19,8 @@
 
 import java.io.IOException;
 import java.io.Serializable;
-import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -28,9 +28,9 @@
 import org.apache.beam.sdk.values.TupleTag;
 
 /**
- * Serializable {@link ExecutionContext.StepContext} that does nothing.
+ * Serializable {@link StepContext} that does nothing.
  */
-public class NoOpStepContext implements ExecutionContext.StepContext, Serializable {
+public class NoOpStepContext implements StepContext, Serializable {
   private static final long serialVersionUID = 1L;
 
   @Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
index 23d61f89b1db..ed3714372b47 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
@@ -49,7 +49,7 @@
  * {@link #getOrCreateStepContext(String, String)}, and {@link #getAllStepContexts()}
  * will be appropriately specialized.
  */
-public abstract class BaseExecutionContext<T extends ExecutionContext.StepContext>
+public abstract class BaseExecutionContext<T extends StepContext>
     implements ExecutionContext {
 
   private Map<String, T> cachedStepContexts = new LinkedHashMap<>();
@@ -81,7 +81,7 @@ public T create() {
    * Factory method interface to create an execution context if none exists during
    * {@link #getOrCreateStepContext(String, CreateStepContextFunction)}.
    */
-  protected interface CreateStepContextFunction<T extends ExecutionContext.StepContext> {
+  protected interface CreateStepContextFunction<T extends org.apache.beam.runners.core.StepContext> {
     T create();
   }
 
@@ -111,12 +111,12 @@ public void noteOutput(WindowedValue<?> output) {}
   public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {}
 
   /**
-   * Base class for implementations of {@link ExecutionContext.StepContext}.
+   * Base class for implementations of {@link org.apache.beam.runners.core.StepContext}.
    *
    * <p>To complete a concrete subclass, implement {@link #timerInternals} and
    * {@link #stateInternals}.
    */
-  public abstract static class StepContext implements ExecutionContext.StepContext {
+  public abstract static class StepContext implements org.apache.beam.runners.core.StepContext {
     private final ExecutionContext executionContext;
     private final String stepName;
     private final String transformName;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
index 71dfd11a496d..9d3e25dbb310 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnRunners.java
@@ -19,7 +19,6 @@
 
 import java.util.Collection;
 import java.util.List;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
 import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn;
 import org.apache.beam.runners.core.StatefulDoFnRunner.CleanupTimer;
 import org.apache.beam.runners.core.StatefulDoFnRunner.StateCleaner;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
index d2fdaac50982..f431c92859ec 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
@@ -17,11 +17,8 @@
  */
 package org.apache.beam.runners.core;
 
-import java.io.IOException;
 import java.util.Collection;
-import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.TupleTag;
 
@@ -52,48 +49,4 @@ public interface ExecutionContext {
    */
   void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
 
-  /**
-   * Per-step, per-key context used for retrieving state.
-   */
-  public interface StepContext {
-
-    /**
-     * The name of the step.
-     */
-    String getStepName();
-
-    /**
-     * The name of the transform for the step.
-     */
-    String getTransformName();
-
-    /**
-     * Hook for subclasses to implement that will be called whenever
-     * {@link WindowedContext#output}
-     * is called.
-     */
-    void noteOutput(WindowedValue<?> output);
-
-    /**
-     * Hook for subclasses to implement that will be called whenever
-     * {@link WindowedContext#output}
-     * is called.
-     */
-    void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
-
-    /**
-     * Writes the given {@code PCollectionView} data to a globally accessible location.
-     */
-    <T, W extends BoundedWindow> void writePCollectionViewData(
-        TupleTag<?> tag,
-        Iterable<WindowedValue<T>> data,
-        Coder<Iterable<WindowedValue<T>>> dataCoder,
-        W window,
-        Coder<W> windowCoder)
-            throws IOException;
-
-    StateInternals stateInternals();
-
-    TimerInternals timerInternals();
-  }
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 65384da9f8f7..adbe62e4cc43 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -29,7 +29,6 @@
 import java.util.Set;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.State;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
new file mode 100644
index 000000000000..a414830bae6d
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import java.io.IOException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Per-step, per-key context used for retrieving state.
+ */
+public interface StepContext {
+
+  /**
+   * The name of the step.
+   */
+  String getStepName();
+
+  /**
+   * The name of the transform for the step.
+   */
+  String getTransformName();
+
+  /**
+   * Hook for subclasses to implement that will be called whenever
+   * {@link WindowedContext#output}
+   * is called.
+   */
+  void noteOutput(WindowedValue<?> output);
+
+  /**
+   * Hook for subclasses to implement that will be called whenever
+   * {@link WindowedContext#output}
+   * is called.
+   */
+  void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
+
+  /**
+   * Writes the given {@code PCollectionView} data to a globally accessible location.
+   */
+  <T, W extends BoundedWindow> void writePCollectionViewData(
+      TupleTag<?> tag,
+      Iterable<WindowedValue<T>> data,
+      Coder<Iterable<WindowedValue<T>>> dataCoder,
+      W window,
+      Coder<W> windowCoder)
+          throws IOException;
+
+  StateInternals stateInternals();
+
+  TimerInternals timerInternals();
+}
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
index 86408016f19b..c394ebdf6f28 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
@@ -18,7 +18,7 @@
 package org.apache.beam.runners.flink.translation.functions;
 
 import java.io.IOException;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.sdk.coders.Coder;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index f35ba7a0b086..c9f106a9079b 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -32,7 +32,6 @@
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.runners.core.DoFnRunners;
-import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetNewDoFn;
 import org.apache.beam.runners.core.NullSideInputReader;
 import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
@@ -184,7 +183,7 @@ public DoFnOperator(
         TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
   }
 
-  private ExecutionContext.StepContext createStepContext() {
+  private org.apache.beam.runners.core.StepContext createStepContext() {
     return new StepContext();
   }
 
@@ -250,7 +249,7 @@ public void open() throws Exception {
 
     doFnInvoker.invokeSetup();
 
-    ExecutionContext.StepContext stepContext = createStepContext();
+    org.apache.beam.runners.core.StepContext stepContext = createStepContext();
 
     doFnRunner = DoFnRunners.simpleRunner(
         serializedOptions.getPipelineOptions(),
@@ -676,7 +675,7 @@ public <T> void output(TupleTag<T> tag, WindowedValue<T> value) {
    * {@link StepContext} for running {@link DoFn DoFns} on Flink. This does not allow
    * accessing state or timer internals.
    */
-  protected class StepContext implements ExecutionContext.StepContext {
+  protected class StepContext implements org.apache.beam.runners.core.StepContext {
 
     @Override
     public String getStepName() {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
index ffe343bc2d8b..91474223a7cb 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
@@ -24,7 +24,7 @@
 import java.util.Iterator;
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.sdk.coders.Coder;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
index 9b79d110bf37..b206bc7b8f85 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
@@ -19,7 +19,7 @@
 package org.apache.beam.fn.harness.fake;
 
 import java.io.IOException;
-import org.apache.beam.runners.core.ExecutionContext.StepContext;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.sdk.coders.Coder;

From bed1c53fb47e2d623d6671ce69b82579992df642 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 15:30:33 -0700
Subject: [PATCH 289/387] Remove StepContext.noteOutput

---
 .../apex/translation/utils/NoOpStepContext.java  |  8 --------
 .../beam/runners/core/BaseExecutionContext.java  | 16 ----------------
 .../beam/runners/core/ExecutionContext.java      | 16 ----------------
 .../beam/runners/core/SimpleDoFnRunner.java      |  6 ------
 .../apache/beam/runners/core/StepContext.java    | 15 ---------------
 .../functions/FlinkNoOpStepContext.java          | 10 ----------
 .../wrappers/streaming/DoFnOperator.java         |  6 ------
 .../spark/translation/SparkProcessContext.java   |  6 ------
 .../beam/fn/harness/fake/FakeStepContext.java    |  8 --------
 9 files changed, 91 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
index 241a9856736d..51e843b56652 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
@@ -43,14 +43,6 @@ public String getTransformName() {
     return null;
   }
 
-  @Override
-  public void noteOutput(WindowedValue<?> output) {
-  }
-
-  @Override
-  public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
-  }
-
   @Override
   public <T, W extends BoundedWindow> void writePCollectionViewData(TupleTag<?> tag,
       Iterable<WindowedValue<T>> data,
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
index ed3714372b47..a00699998bcd 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
@@ -104,12 +104,6 @@ public Collection<? extends T> getAllStepContexts() {
     return Collections.unmodifiableCollection(cachedStepContexts.values());
   }
 
-  @Override
-  public void noteOutput(WindowedValue<?> output) {}
-
-  @Override
-  public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {}
-
   /**
    * Base class for implementations of {@link org.apache.beam.runners.core.StepContext}.
    *
@@ -137,16 +131,6 @@ public String getTransformName() {
       return transformName;
     }
 
-    @Override
-    public void noteOutput(WindowedValue<?> output) {
-      executionContext.noteOutput(output);
-    }
-
-    @Override
-    public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
-      executionContext.noteOutput(tag, output);
-    }
-
     @Override
     public <T, W extends BoundedWindow> void writePCollectionViewData(
         TupleTag<?> tag,
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
index f431c92859ec..eac3599d9757 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
@@ -18,9 +18,6 @@
 package org.apache.beam.runners.core;
 
 import java.util.Collection;
-import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
 
 /**
  * Context for the current execution. This is guaranteed to exist during processing,
@@ -36,17 +33,4 @@ public interface ExecutionContext {
    * Returns a collection view of all of the {@link StepContext}s.
    */
   Collection<? extends StepContext> getAllStepContexts();
-
-  /**
-   * Hook for subclasses to implement that will be called whenever
-   * {@link WindowedContext#output(TupleTag, Object)} is called.
-   */
-  void noteOutput(WindowedValue<?> output);
-
-  /**
-   * Hook for subclasses to implement that will be called whenever
-   * {@link WindowedContext#output(TupleTag, Object)} is called.
-   */
-  void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
-
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index adbe62e4cc43..97b0b3307740 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -352,9 +352,6 @@ void outputWindowedValue(
 
     void outputWindowedValue(WindowedValue<OutputT> windowedElem) {
       outputManager.output(mainOutputTag, windowedElem);
-      if (stepContext != null) {
-        stepContext.noteOutput(windowedElem);
-      }
     }
 
     private <T> void outputWindowedValue(
@@ -380,9 +377,6 @@ private <T> void outputWindowedValue(TupleTag<T> tag, WindowedValue<T> windowedE
       }
 
       outputManager.output(tag, windowedElem);
-      if (stepContext != null) {
-        stepContext.noteOutput(tag, windowedElem);
-      }
     }
   }
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
index a414830bae6d..fd2575d5f74b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
@@ -19,7 +19,6 @@
 
 import java.io.IOException;
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.TupleTag;
@@ -39,20 +38,6 @@ public interface StepContext {
    */
   String getTransformName();
 
-  /**
-   * Hook for subclasses to implement that will be called whenever
-   * {@link WindowedContext#output}
-   * is called.
-   */
-  void noteOutput(WindowedValue<?> output);
-
-  /**
-   * Hook for subclasses to implement that will be called whenever
-   * {@link WindowedContext#output}
-   * is called.
-   */
-  void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
-
   /**
    * Writes the given {@code PCollectionView} data to a globally accessible location.
    */
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
index c394ebdf6f28..d999494eee79 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
@@ -41,16 +41,6 @@ public String getTransformName() {
     return null;
   }
 
-  @Override
-  public void noteOutput(WindowedValue<?> output) {
-
-  }
-
-  @Override
-  public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
-
-  }
-
   @Override
   public <T, W extends BoundedWindow> void writePCollectionViewData(
       TupleTag<?> tag,
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index c9f106a9079b..2bb9c2003898 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -687,12 +687,6 @@ public String getTransformName() {
       return null;
     }
 
-    @Override
-    public void noteOutput(WindowedValue<?> output) {}
-
-    @Override
-    public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {}
-
     @Override
     public <T, W extends BoundedWindow> void writePCollectionViewData(
         TupleTag<?> tag,
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
index 91474223a7cb..31e616cbf0f4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
@@ -109,12 +109,6 @@ public String getTransformName() {
       return null;
     }
 
-    @Override
-    public void noteOutput(WindowedValue<?> output) { }
-
-    @Override
-    public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) { }
-
     @Override
     public <T, W extends BoundedWindow> void writePCollectionViewData(
         TupleTag<?> tag,
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
index b206bc7b8f85..750c167d8993 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
@@ -41,14 +41,6 @@ public String getTransformName() {
     return "TODO";
   }
 
-  @Override
-  public void noteOutput(WindowedValue<?> output) {
-  }
-
-  @Override
-  public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
-  }
-
   @Override
   public <T, W extends BoundedWindow> void writePCollectionViewData(
       TupleTag<?> tag,

From 8b7a1f6dfe0ac33814a0b0c67f37f47ab449ec4b Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 15:34:37 -0700
Subject: [PATCH 290/387] Rename BaseExecutionContext.StepContext to
 BaseStepContext

---
 .../beam/runners/core/BaseExecutionContext.java    | 14 +++++++-------
 .../beam/runners/core/SimpleDoFnRunnerTest.java    |  5 +++--
 .../beam/runners/core/StatefulDoFnRunnerTest.java  |  5 +++--
 .../runners/direct/DirectExecutionContext.java     |  2 +-
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
index a00699998bcd..5667250b3288 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
@@ -31,11 +31,11 @@
  * Base class for implementations of {@link ExecutionContext}.
  *
  * <p>A concrete subclass should implement {@link #createStepContext} to create the appropriate
- * {@link StepContext} implementation. Any {@code StepContext} created will
+ * {@link BaseStepContext} implementation. Any {@code StepContext} created will
  * be cached for the lifetime of this {@link ExecutionContext}.
  *
  * <p>BaseExecutionContext is generic to allow implementing subclasses to return a concrete subclass
- * of {@link StepContext} from {@link #getOrCreateStepContext(String, String)} and
+ * of {@link BaseStepContext} from {@link #getOrCreateStepContext(String, String)} and
  * {@link #getAllStepContexts()} without forcing each subclass to override the method, e.g.
  * <pre>{@code
  * {@literal @}Override
@@ -56,12 +56,12 @@ public abstract class BaseExecutionContext<T extends StepContext>
 
   /**
    * Implementations should override this to create the specific type
-   * of {@link StepContext} they need.
+   * of {@link BaseStepContext} they need.
    */
   protected abstract T createStepContext(String stepName, String transformName);
 
   /**
-   * Returns the {@link StepContext} associated with the given step.
+   * Returns the {@link BaseStepContext} associated with the given step.
    */
   @Override
   public T getOrCreateStepContext(String stepName, String transformName) {
@@ -97,7 +97,7 @@ protected final T getOrCreateStepContext(String stepName,
   }
 
   /**
-   * Returns a collection view of all of the {@link StepContext}s.
+   * Returns a collection view of all of the {@link BaseStepContext}s.
    */
   @Override
   public Collection<? extends T> getAllStepContexts() {
@@ -110,12 +110,12 @@ public Collection<? extends T> getAllStepContexts() {
    * <p>To complete a concrete subclass, implement {@link #timerInternals} and
    * {@link #stateInternals}.
    */
-  public abstract static class StepContext implements org.apache.beam.runners.core.StepContext {
+  public abstract static class BaseStepContext implements org.apache.beam.runners.core.StepContext {
     private final ExecutionContext executionContext;
     private final String stepName;
     private final String transformName;
 
-    public StepContext(ExecutionContext executionContext, String stepName, String transformName) {
+    public BaseStepContext(ExecutionContext executionContext, String stepName, String transformName) {
       this.executionContext = executionContext;
       this.stepName = stepName;
       this.transformName = transformName;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index abefd1c488b7..3750e6c9f0f7 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -29,7 +29,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import org.apache.beam.runners.core.BaseExecutionContext.StepContext;
+import org.apache.beam.runners.core.BaseExecutionContext.BaseStepContext;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.sdk.coders.Coder;
@@ -63,7 +63,8 @@
 public class SimpleDoFnRunnerTest {
   @Rule public ExpectedException thrown = ExpectedException.none();
 
-  @Mock StepContext mockStepContext;
+  @Mock
+  BaseStepContext mockStepContext;
 
   @Mock TimerInternals mockTimerInternals;
 
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index 5172f433135a..a335c3a5daa4 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -24,7 +24,7 @@
 
 import com.google.common.base.MoreObjects;
 import java.util.Collections;
-import org.apache.beam.runners.core.BaseExecutionContext.StepContext;
+import org.apache.beam.runners.core.BaseExecutionContext.BaseStepContext;
 import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
@@ -69,7 +69,8 @@ public class StatefulDoFnRunnerTest {
   private static final IntervalWindow WINDOW_2 =
       new IntervalWindow(new Instant(10), new Instant(20));
 
-  @Mock StepContext mockStepContext;
+  @Mock
+  BaseStepContext mockStepContext;
 
   private InMemoryStateInternals<String> stateInternals;
   private InMemoryTimerInternals timerInternals;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 107f39ad4f74..6d2d02ab4fb2 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -57,7 +57,7 @@ protected DirectStepContext createStepContext(String stepName, String transformN
    * Step Context for the {@link DirectRunner}.
    */
   public class DirectStepContext
-      extends BaseExecutionContext.StepContext {
+      extends BaseStepContext {
     private CopyOnAccessInMemoryStateInternals<?> stateInternals;
     private DirectTimerInternals timerInternals;
 

From 59322d51e80e7480710a296f51a4cb65303f5e06 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 15:35:46 -0700
Subject: [PATCH 291/387] Move BaseStepContext to the top level

---
 .../runners/core/BaseExecutionContext.java    | 46 -------------
 .../beam/runners/core/BaseStepContext.java    | 66 +++++++++++++++++++
 .../runners/core/SimpleDoFnRunnerTest.java    |  1 -
 .../runners/core/StatefulDoFnRunnerTest.java  |  1 -
 .../direct/DirectExecutionContext.java        |  1 +
 5 files changed, 67 insertions(+), 48 deletions(-)
 create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
index 5667250b3288..877fa0a4f105 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
@@ -17,15 +17,10 @@
  */
 package org.apache.beam.runners.core;
 
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
 
 /**
  * Base class for implementations of {@link ExecutionContext}.
@@ -104,45 +99,4 @@ public Collection<? extends T> getAllStepContexts() {
     return Collections.unmodifiableCollection(cachedStepContexts.values());
   }
 
-  /**
-   * Base class for implementations of {@link org.apache.beam.runners.core.StepContext}.
-   *
-   * <p>To complete a concrete subclass, implement {@link #timerInternals} and
-   * {@link #stateInternals}.
-   */
-  public abstract static class BaseStepContext implements org.apache.beam.runners.core.StepContext {
-    private final ExecutionContext executionContext;
-    private final String stepName;
-    private final String transformName;
-
-    public BaseStepContext(ExecutionContext executionContext, String stepName, String transformName) {
-      this.executionContext = executionContext;
-      this.stepName = stepName;
-      this.transformName = transformName;
-    }
-
-    @Override
-    public String getStepName() {
-      return stepName;
-    }
-
-    @Override
-    public String getTransformName() {
-      return transformName;
-    }
-
-    @Override
-    public <T, W extends BoundedWindow> void writePCollectionViewData(
-        TupleTag<?> tag,
-        Iterable<WindowedValue<T>> data, Coder<Iterable<WindowedValue<T>>> dataCoder,
-        W window, Coder<W> windowCoder) throws IOException {
-      throw new UnsupportedOperationException("Not implemented.");
-    }
-
-    @Override
-    public abstract StateInternals stateInternals();
-
-    @Override
-    public abstract TimerInternals timerInternals();
-  }
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
new file mode 100644
index 000000000000..f0436acefdc5
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import java.io.IOException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Base class for implementations of {@link StepContext}.
+ *
+ * <p>To complete a concrete subclass, implement {@link #timerInternals} and
+ * {@link #stateInternals}.
+ */
+public abstract class BaseStepContext implements StepContext {
+  private final ExecutionContext executionContext;
+  private final String stepName;
+  private final String transformName;
+
+  public BaseStepContext(ExecutionContext executionContext, String stepName, String transformName) {
+    this.executionContext = executionContext;
+    this.stepName = stepName;
+    this.transformName = transformName;
+  }
+
+  @Override
+  public String getStepName() {
+    return stepName;
+  }
+
+  @Override
+  public String getTransformName() {
+    return transformName;
+  }
+
+  @Override
+  public <T, W extends BoundedWindow> void writePCollectionViewData(
+      TupleTag<?> tag,
+      Iterable<WindowedValue<T>> data, Coder<Iterable<WindowedValue<T>>> dataCoder,
+      W window, Coder<W> windowCoder) throws IOException {
+    throw new UnsupportedOperationException("Not implemented.");
+  }
+
+  @Override
+  public abstract StateInternals stateInternals();
+
+  @Override
+  public abstract TimerInternals timerInternals();
+}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index 3750e6c9f0f7..59e5857c3aa1 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -29,7 +29,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import org.apache.beam.runners.core.BaseExecutionContext.BaseStepContext;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
 import org.apache.beam.sdk.coders.Coder;
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index a335c3a5daa4..62a657882cc6 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -24,7 +24,6 @@
 
 import com.google.common.base.MoreObjects;
 import java.util.Collections;
-import org.apache.beam.runners.core.BaseExecutionContext.BaseStepContext;
 import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 6d2d02ab4fb2..e5b88e5ac8ee 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -18,6 +18,7 @@
 package org.apache.beam.runners.direct;
 
 import org.apache.beam.runners.core.BaseExecutionContext;
+import org.apache.beam.runners.core.BaseStepContext;
 import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;

From 248c808a6603dc2c28a0b55296e0d596b8903a08 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 15:36:41 -0700
Subject: [PATCH 292/387] Remove extraneous ExecutionContext parameter to
 BaseStepContext

---
 .../java/org/apache/beam/runners/core/BaseStepContext.java    | 4 +---
 .../apache/beam/runners/direct/DirectExecutionContext.java    | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
index f0436acefdc5..014fe0def371 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
@@ -30,12 +30,10 @@
  * {@link #stateInternals}.
  */
 public abstract class BaseStepContext implements StepContext {
-  private final ExecutionContext executionContext;
   private final String stepName;
   private final String transformName;
 
-  public BaseStepContext(ExecutionContext executionContext, String stepName, String transformName) {
-    this.executionContext = executionContext;
+  public BaseStepContext(String stepName, String transformName) {
     this.stepName = stepName;
     this.transformName = transformName;
   }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index e5b88e5ac8ee..d676f247298c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -64,7 +64,7 @@ public class DirectStepContext
 
     public DirectStepContext(
         ExecutionContext executionContext, String stepName, String transformName) {
-      super(executionContext, stepName, transformName);
+      super(stepName, transformName);
     }
 
     @Override

From 5ac24e0a89b95feafccbe381bdde9c11fdf82a88 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 15:44:17 -0700
Subject: [PATCH 293/387] Implement StepContext directly in the DirectRunner

---
 .../direct/DirectExecutionContext.java        | 33 ++++++++++++++++---
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index d676f247298c..2a75ef51a4ba 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -17,13 +17,18 @@
  */
 package org.apache.beam.runners.direct;
 
+import java.io.IOException;
 import org.apache.beam.runners.core.BaseExecutionContext;
-import org.apache.beam.runners.core.BaseStepContext;
 import org.apache.beam.runners.core.ExecutionContext;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.TupleTag;
 
 /**
  * Execution Context for the {@link DirectRunner}.
@@ -57,14 +62,16 @@ protected DirectStepContext createStepContext(String stepName, String transformN
   /**
    * Step Context for the {@link DirectRunner}.
    */
-  public class DirectStepContext
-      extends BaseStepContext {
+  public class DirectStepContext implements StepContext {
     private CopyOnAccessInMemoryStateInternals<?> stateInternals;
     private DirectTimerInternals timerInternals;
+    private final String stepName;
+    private final String transformName;
 
     public DirectStepContext(
         ExecutionContext executionContext, String stepName, String transformName) {
-      super(stepName, transformName);
+      this.stepName = stepName;
+      this.transformName = transformName;
     }
 
     @Override
@@ -95,6 +102,24 @@ public CopyOnAccessInMemoryStateInternals commitState() {
       return null;
     }
 
+    @Override
+    public String getStepName() {
+      return stepName;
+    }
+
+    @Override
+    public String getTransformName() {
+      return transformName;
+    }
+
+    @Override
+    public <T, W extends BoundedWindow> void writePCollectionViewData(
+        TupleTag<?> tag,
+        Iterable<WindowedValue<T>> data, Coder<Iterable<WindowedValue<T>>> dataCoder,
+        W window, Coder<W> windowCoder) throws IOException {
+      throw new UnsupportedOperationException("Not implemented.");
+    }
+
     /**
      * Gets the timer update of the {@link TimerInternals} of this {@link DirectStepContext},
      * which is empty if the {@link TimerInternals} were never accessed.

From 32c6cb160f42e401f3e170cc8ed18d76c627d3e4 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 16:26:00 -0700
Subject: [PATCH 294/387] Remove writePCollectionViewData from the Beam
 codebase

---
 .../apex/translation/utils/NoOpStepContext.java | 13 -------------
 .../beam/runners/core/BaseStepContext.java      | 14 --------------
 .../apache/beam/runners/core/StepContext.java   | 17 -----------------
 .../runners/direct/DirectExecutionContext.java  | 13 -------------
 .../functions/FlinkNoOpStepContext.java         | 16 +---------------
 .../wrappers/streaming/DoFnOperator.java        | 11 -----------
 .../spark/translation/SparkProcessContext.java  | 14 +-------------
 .../beam/fn/harness/fake/FakeStepContext.java   | 16 +---------------
 8 files changed, 3 insertions(+), 111 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
index 51e843b56652..820b1897b5db 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
@@ -17,15 +17,10 @@
  */
 package org.apache.beam.runners.apex.translation.utils;
 
-import java.io.IOException;
 import java.io.Serializable;
 import org.apache.beam.runners.core.StateInternals;
 import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
 
 /**
  * Serializable {@link StepContext} that does nothing.
@@ -43,14 +38,6 @@ public String getTransformName() {
     return null;
   }
 
-  @Override
-  public <T, W extends BoundedWindow> void writePCollectionViewData(TupleTag<?> tag,
-      Iterable<WindowedValue<T>> data,
-      Coder<Iterable<WindowedValue<T>>> dataCoder, W window, Coder<W> windowCoder) throws
-      IOException {
-
-  }
-
   @Override
   public StateInternals stateInternals() {
     return null;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
index 014fe0def371..e639c46c9f62 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
@@ -17,12 +17,6 @@
  */
 package org.apache.beam.runners.core;
 
-import java.io.IOException;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-
 /**
  * Base class for implementations of {@link StepContext}.
  *
@@ -48,14 +42,6 @@ public String getTransformName() {
     return transformName;
   }
 
-  @Override
-  public <T, W extends BoundedWindow> void writePCollectionViewData(
-      TupleTag<?> tag,
-      Iterable<WindowedValue<T>> data, Coder<Iterable<WindowedValue<T>>> dataCoder,
-      W window, Coder<W> windowCoder) throws IOException {
-    throw new UnsupportedOperationException("Not implemented.");
-  }
-
   @Override
   public abstract StateInternals stateInternals();
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
index fd2575d5f74b..62a81f154eca 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
@@ -17,12 +17,6 @@
  */
 package org.apache.beam.runners.core;
 
-import java.io.IOException;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-
 /**
  * Per-step, per-key context used for retrieving state.
  */
@@ -38,17 +32,6 @@ public interface StepContext {
    */
   String getTransformName();
 
-  /**
-   * Writes the given {@code PCollectionView} data to a globally accessible location.
-   */
-  <T, W extends BoundedWindow> void writePCollectionViewData(
-      TupleTag<?> tag,
-      Iterable<WindowedValue<T>> data,
-      Coder<Iterable<WindowedValue<T>>> dataCoder,
-      W window,
-      Coder<W> windowCoder)
-          throws IOException;
-
   StateInternals stateInternals();
 
   TimerInternals timerInternals();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 2a75ef51a4ba..39174d630954 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -17,7 +17,6 @@
  */
 package org.apache.beam.runners.direct;
 
-import java.io.IOException;
 import org.apache.beam.runners.core.BaseExecutionContext;
 import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.StepContext;
@@ -25,10 +24,6 @@
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
 
 /**
  * Execution Context for the {@link DirectRunner}.
@@ -112,14 +107,6 @@ public String getTransformName() {
       return transformName;
     }
 
-    @Override
-    public <T, W extends BoundedWindow> void writePCollectionViewData(
-        TupleTag<?> tag,
-        Iterable<WindowedValue<T>> data, Coder<Iterable<WindowedValue<T>>> dataCoder,
-        W window, Coder<W> windowCoder) throws IOException {
-      throw new UnsupportedOperationException("Not implemented.");
-    }
-
     /**
      * Gets the timer update of the {@link TimerInternals} of this {@link DirectStepContext},
      * which is empty if the {@link TimerInternals} were never accessed.
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
index d999494eee79..1ff322ee0ba5 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
@@ -17,14 +17,9 @@
  */
 package org.apache.beam.runners.flink.translation.functions;
 
-import java.io.IOException;
-import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
 
 /**
  * A {@link StepContext} for Flink Batch Runner execution.
@@ -41,15 +36,6 @@ public String getTransformName() {
     return null;
   }
 
-  @Override
-  public <T, W extends BoundedWindow> void writePCollectionViewData(
-      TupleTag<?> tag,
-      Iterable<WindowedValue<T>> data,
-      Coder<Iterable<WindowedValue<T>>> dataCoder,
-      W window,
-      Coder<W> windowCoder) throws IOException {
-  }
-
   @Override
   public StateInternals stateInternals() {
     return null;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index 2bb9c2003898..4f8998e9df8c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -23,7 +23,6 @@
 import com.google.common.collect.Iterables;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
-import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -687,16 +686,6 @@ public String getTransformName() {
       return null;
     }
 
-    @Override
-    public <T, W extends BoundedWindow> void writePCollectionViewData(
-        TupleTag<?> tag,
-        Iterable<WindowedValue<T>> data,
-        Coder<Iterable<WindowedValue<T>>> dataCoder,
-        W window,
-        Coder<W> windowCoder) throws IOException {
-      throw new UnsupportedOperationException("Writing side-input data is not supported.");
-    }
-
     @Override
     public StateInternals stateInternals() {
       return stateInternals;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
index 31e616cbf0f4..e693143ade16 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
@@ -20,19 +20,15 @@
 
 import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.Lists;
-import java.io.IOException;
 import java.util.Iterator;
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.runners.core.DoFnRunners.OutputManager;
-import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
 
 
 /**
@@ -109,14 +105,6 @@ public String getTransformName() {
       return null;
     }
 
-    @Override
-    public <T, W extends BoundedWindow> void writePCollectionViewData(
-        TupleTag<?> tag,
-        Iterable<WindowedValue<T>> data,
-        Coder<Iterable<WindowedValue<T>>> dataCoder,
-        W window,
-        Coder<W> windowCoder) throws IOException { }
-
     @Override
     public StateInternals stateInternals() {
       return null;
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
index 750c167d8993..3f6a2daed61c 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
@@ -18,14 +18,9 @@
 
 package org.apache.beam.fn.harness.fake;
 
-import java.io.IOException;
-import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
 
 /**
  * A fake {@link StepContext} factory that performs no-ops.
@@ -41,15 +36,6 @@ public String getTransformName() {
     return "TODO";
   }
 
-  @Override
-  public <T, W extends BoundedWindow> void writePCollectionViewData(
-      TupleTag<?> tag,
-      Iterable<WindowedValue<T>> data,
-      Coder<Iterable<WindowedValue<T>>> dataCoder,
-      W window,
-      Coder<W> windowCoder) throws IOException {
-  }
-
   @Override
   public StateInternals stateInternals() {
     throw new UnsupportedOperationException();

From 0be3cf3462c19f0b007b2329c95ea4865d22cad5 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 16:50:41 -0700
Subject: [PATCH 295/387] Inline and delete BaseExecutionContext

---
 .../runners/core/BaseExecutionContext.java    | 102 ------------------
 .../direct/DirectExecutionContext.java        |  39 +++++--
 2 files changed, 32 insertions(+), 109 deletions(-)
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
deleted file mode 100644
index 877fa0a4f105..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-/**
- * Base class for implementations of {@link ExecutionContext}.
- *
- * <p>A concrete subclass should implement {@link #createStepContext} to create the appropriate
- * {@link BaseStepContext} implementation. Any {@code StepContext} created will
- * be cached for the lifetime of this {@link ExecutionContext}.
- *
- * <p>BaseExecutionContext is generic to allow implementing subclasses to return a concrete subclass
- * of {@link BaseStepContext} from {@link #getOrCreateStepContext(String, String)} and
- * {@link #getAllStepContexts()} without forcing each subclass to override the method, e.g.
- * <pre>{@code
- * {@literal @}Override
- * StreamingModeExecutionContext.StepContext getOrCreateStepContext(...) {
- *   return (StreamingModeExecutionContext.StepContext) super.getOrCreateStepContext(...);
- * }
- * }</pre>
- *
- * <p>When a subclass of {@code BaseExecutionContext} has been downcast, the return types of
- * {@link #createStepContext(String, String)},
- * {@link #getOrCreateStepContext(String, String)}, and {@link #getAllStepContexts()}
- * will be appropriately specialized.
- */
-public abstract class BaseExecutionContext<T extends StepContext>
-    implements ExecutionContext {
-
-  private Map<String, T> cachedStepContexts = new LinkedHashMap<>();
-
-  /**
-   * Implementations should override this to create the specific type
-   * of {@link BaseStepContext} they need.
-   */
-  protected abstract T createStepContext(String stepName, String transformName);
-
-  /**
-   * Returns the {@link BaseStepContext} associated with the given step.
-   */
-  @Override
-  public T getOrCreateStepContext(String stepName, String transformName) {
-    final String finalStepName = stepName;
-    final String finalTransformName = transformName;
-    return getOrCreateStepContext(
-        stepName,
-        new CreateStepContextFunction<T>() {
-          @Override
-          public T create() {
-            return createStepContext(finalStepName, finalTransformName);
-          }
-        });
-  }
-
-  /**
-   * Factory method interface to create an execution context if none exists during
-   * {@link #getOrCreateStepContext(String, CreateStepContextFunction)}.
-   */
-  protected interface CreateStepContextFunction<T extends org.apache.beam.runners.core.StepContext> {
-    T create();
-  }
-
-  protected final T getOrCreateStepContext(String stepName,
-      CreateStepContextFunction<T> createContextFunc) {
-    T context = cachedStepContexts.get(stepName);
-    if (context == null) {
-      context = createContextFunc.create();
-      cachedStepContexts.put(stepName, context);
-    }
-
-    return context;
-  }
-
-  /**
-   * Returns a collection view of all of the {@link BaseStepContext}s.
-   */
-  @Override
-  public Collection<? extends T> getAllStepContexts() {
-    return Collections.unmodifiableCollection(cachedStepContexts.values());
-  }
-
-}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 39174d630954..9b6866216a61 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -17,11 +17,14 @@
  */
 package org.apache.beam.runners.direct;
 
-import org.apache.beam.runners.core.BaseExecutionContext;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.apache.beam.runners.core.BaseStepContext;
 import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
 import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
 
@@ -31,12 +34,12 @@
  * <p>This implementation is not thread safe. A new {@link DirectExecutionContext} must be created
  * for each thread that requires it.
  */
-class DirectExecutionContext
-    extends BaseExecutionContext<DirectStepContext> {
+class DirectExecutionContext implements ExecutionContext {
   private final Clock clock;
   private final StructuralKey<?> key;
   private final CopyOnAccessInMemoryStateInternals existingState;
   private final TransformWatermarks watermarks;
+  private Map<String, DirectStepContext> cachedStepContexts = new LinkedHashMap<>();
 
   public DirectExecutionContext(
       Clock clock,
@@ -49,9 +52,31 @@ public DirectExecutionContext(
     this.watermarks = watermarks;
   }
 
+  private DirectStepContext createStepContext(String stepName, String transformName) {
+    return new DirectStepContext(stepName, transformName);
+  }
+
+  /**
+   * Returns the {@link BaseStepContext} associated with the given step.
+   */
+  @Override
+  public DirectStepContext getOrCreateStepContext(String stepName, String transformName) {
+    final String finalStepName = stepName;
+    final String finalTransformName = transformName;
+    DirectStepContext context = cachedStepContexts.get(stepName);
+    if (context == null) {
+      context = createStepContext(finalStepName, finalTransformName);
+      cachedStepContexts.put(stepName, context);
+    }
+    return context;
+  }
+
+  /**
+   * Returns a collection view of all of the {@link BaseStepContext}s.
+   */
   @Override
-  protected DirectStepContext createStepContext(String stepName, String transformName) {
-    return new DirectStepContext(this, stepName, transformName);
+  public Collection<? extends DirectStepContext> getAllStepContexts() {
+    return Collections.unmodifiableCollection(cachedStepContexts.values());
   }
 
   /**
@@ -64,7 +89,7 @@ public class DirectStepContext implements StepContext {
     private final String transformName;
 
     public DirectStepContext(
-        ExecutionContext executionContext, String stepName, String transformName) {
+        String stepName, String transformName) {
       this.stepName = stepName;
       this.transformName = transformName;
     }

From 97c230af62151fdbe06ac622282d69c74db30b2f Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 17:33:38 -0700
Subject: [PATCH 296/387] Delete unused ExecutionContext

---
 .../beam/runners/core/ExecutionContext.java   | 36 -------------------
 .../direct/DirectExecutionContext.java        |  3 +-
 .../runners/direct/EvaluationContext.java     | 29 +++++++--------
 3 files changed, 14 insertions(+), 54 deletions(-)
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
deleted file mode 100644
index eac3599d9757..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-import java.util.Collection;
-
-/**
- * Context for the current execution. This is guaranteed to exist during processing,
- * but does not necessarily persist between different batches of work.
- */
-public interface ExecutionContext {
-  /**
-   * Returns the {@link StepContext} associated with the given step.
-   */
-  StepContext getOrCreateStepContext(String stepName, String transformName);
-
-  /**
-   * Returns a collection view of all of the {@link StepContext}s.
-   */
-  Collection<? extends StepContext> getAllStepContexts();
-}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 9b6866216a61..05dbebc053c7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -22,7 +22,6 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import org.apache.beam.runners.core.BaseStepContext;
-import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
@@ -34,7 +33,7 @@
  * <p>This implementation is not thread safe. A new {@link DirectExecutionContext} must be created
  * for each thread that requires it.
  */
-class DirectExecutionContext implements ExecutionContext {
+class DirectExecutionContext {
   private final Clock clock;
   private final StructuralKey<?> key;
   private final CopyOnAccessInMemoryStateInternals existingState;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
index c62711996648..88ce85a7cd22 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -31,7 +31,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import javax.annotation.Nullable;
-import org.apache.beam.runners.core.ExecutionContext;
 import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
 import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
@@ -52,22 +51,20 @@
 import org.joda.time.Instant;
 
 /**
- * The evaluation context for a specific pipeline being executed by the
- * {@link DirectRunner}. Contains state shared within the execution across all
- * transforms.
+ * The evaluation context for a specific pipeline being executed by the {@link DirectRunner}.
+ * Contains state shared within the execution across all transforms.
  *
- * <p>{@link EvaluationContext} contains shared state for an execution of the
- * {@link DirectRunner} that can be used while evaluating a {@link PTransform}. This
- * consists of views into underlying state and watermark implementations, access to read and write
- * {@link PCollectionView PCollectionViews}, and managing the
- * {@link ExecutionContext ExecutionContexts}. This includes executing callbacks asynchronously when
- * state changes to the appropriate point (e.g. when a {@link PCollectionView} is requested and
- * known to be empty).
+ * <p>{@link EvaluationContext} contains shared state for an execution of the {@link DirectRunner}
+ * that can be used while evaluating a {@link PTransform}. This consists of views into underlying
+ * state and watermark implementations, access to read and write {@link PCollectionView
+ * PCollectionViews}, and managing the {@link DirectExecutionContext ExecutionContexts}. This
+ * includes executing callbacks asynchronously when state changes to the appropriate point (e.g.
+ * when a {@link PCollectionView} is requested and known to be empty).
  *
- * <p>{@link EvaluationContext} also handles results by committing finalizing bundles based
- * on the current global state and updating the global state appropriately. This includes updating
- * the per-{@link StepAndKey} state, updating global watermarks, and executing any callbacks that
- * can be executed.
+ * <p>{@link EvaluationContext} also handles results by committing finalizing bundles based on the
+ * current global state and updating the global state appropriately. This includes updating the
+ * per-{@link StepAndKey} state, updating global watermarks, and executing any callbacks that can be
+ * executed.
  */
 class EvaluationContext {
   /**
@@ -312,7 +309,7 @@ public DirectOptions getPipelineOptions() {
   }
 
   /**
-   * Get an {@link ExecutionContext} for the provided {@link AppliedPTransform} and key.
+   * Get a {@link DirectExecutionContext} for the provided {@link AppliedPTransform} and key.
    */
   public DirectExecutionContext getExecutionContext(
       AppliedPTransform<?, ?, ?> application, StructuralKey<?> key) {

From acce24ce1388b7953fbb9d87da5bb2271286c58a Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 17:42:10 -0700
Subject: [PATCH 297/387] Remove unused StepContext name methods

---
 .../apex/translation/utils/NoOpStepContext.java        | 10 ----------
 .../org/apache/beam/runners/core/BaseStepContext.java  | 10 ----------
 .../java/org/apache/beam/runners/core/StepContext.java | 10 ----------
 .../beam/runners/direct/DirectExecutionContext.java    | 10 ----------
 .../translation/functions/FlinkNoOpStepContext.java    | 10 ----------
 .../translation/wrappers/streaming/DoFnOperator.java   | 10 ----------
 .../runners/spark/translation/SparkProcessContext.java |  9 ---------
 .../apache/beam/fn/harness/fake/FakeStepContext.java   |  9 ---------
 8 files changed, 78 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
index 820b1897b5db..b49e4da27bcd 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/utils/NoOpStepContext.java
@@ -28,16 +28,6 @@
 public class NoOpStepContext implements StepContext, Serializable {
   private static final long serialVersionUID = 1L;
 
-  @Override
-  public String getStepName() {
-    return null;
-  }
-
-  @Override
-  public String getTransformName() {
-    return null;
-  }
-
   @Override
   public StateInternals stateInternals() {
     return null;
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
index e639c46c9f62..4abd4d228160 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
@@ -32,16 +32,6 @@ public BaseStepContext(String stepName, String transformName) {
     this.transformName = transformName;
   }
 
-  @Override
-  public String getStepName() {
-    return stepName;
-  }
-
-  @Override
-  public String getTransformName() {
-    return transformName;
-  }
-
   @Override
   public abstract StateInternals stateInternals();
 
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
index 62a81f154eca..60fc40291452 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
@@ -22,16 +22,6 @@
  */
 public interface StepContext {
 
-  /**
-   * The name of the step.
-   */
-  String getStepName();
-
-  /**
-   * The name of the transform for the step.
-   */
-  String getTransformName();
-
   StateInternals stateInternals();
 
   TimerInternals timerInternals();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 05dbebc053c7..651af8f4f818 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -121,16 +121,6 @@ public CopyOnAccessInMemoryStateInternals commitState() {
       return null;
     }
 
-    @Override
-    public String getStepName() {
-      return stepName;
-    }
-
-    @Override
-    public String getTransformName() {
-      return transformName;
-    }
-
     /**
      * Gets the timer update of the {@link TimerInternals} of this {@link DirectStepContext},
      * which is empty if the {@link TimerInternals} were never accessed.
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
index 1ff322ee0ba5..9c7b63677ba3 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNoOpStepContext.java
@@ -26,16 +26,6 @@
  */
 public class FlinkNoOpStepContext implements StepContext {
 
-  @Override
-  public String getStepName() {
-    return null;
-  }
-
-  @Override
-  public String getTransformName() {
-    return null;
-  }
-
   @Override
   public StateInternals stateInternals() {
     return null;
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
index 4f8998e9df8c..d2ab7e1cabbd 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
@@ -676,16 +676,6 @@ public <T> void output(TupleTag<T> tag, WindowedValue<T> value) {
    */
   protected class StepContext implements org.apache.beam.runners.core.StepContext {
 
-    @Override
-    public String getStepName() {
-      return null;
-    }
-
-    @Override
-    public String getTransformName() {
-      return null;
-    }
-
     @Override
     public StateInternals stateInternals() {
       return stateInternals;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
index e693143ade16..f4ab7d9b02cb 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
@@ -95,15 +95,6 @@ interface SparkOutputManager<T> extends OutputManager, Iterable<T> {
   }
 
   static class NoOpStepContext implements StepContext {
-    @Override
-    public String getStepName() {
-      return null;
-    }
-
-    @Override
-    public String getTransformName() {
-      return null;
-    }
 
     @Override
     public StateInternals stateInternals() {
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
index 3f6a2daed61c..bdf138b31523 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/fake/FakeStepContext.java
@@ -26,15 +26,6 @@
  * A fake {@link StepContext} factory that performs no-ops.
  */
 public class FakeStepContext implements StepContext {
-  @Override
-  public String getStepName() {
-    return "TODO";
-  }
-
-  @Override
-  public String getTransformName() {
-    return "TODO";
-  }
 
   @Override
   public StateInternals stateInternals() {

From dc585510e6b4def3a0442114d77e96f2b5d4880f Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 17:46:12 -0700
Subject: [PATCH 298/387] Delete unused remnants in DirectExecutionContext

---
 .../beam/runners/direct/DirectExecutionContext.java   | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 651af8f4f818..845256539eb9 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -56,9 +56,8 @@ private DirectStepContext createStepContext(String stepName, String transformNam
   }
 
   /**
-   * Returns the {@link BaseStepContext} associated with the given step.
+   * Returns the {@link StepContext} associated with the given step.
    */
-  @Override
   public DirectStepContext getOrCreateStepContext(String stepName, String transformName) {
     final String finalStepName = stepName;
     final String finalTransformName = transformName;
@@ -70,14 +69,6 @@ public DirectStepContext getOrCreateStepContext(String stepName, String transfor
     return context;
   }
 
-  /**
-   * Returns a collection view of all of the {@link BaseStepContext}s.
-   */
-  @Override
-  public Collection<? extends DirectStepContext> getAllStepContexts() {
-    return Collections.unmodifiableCollection(cachedStepContexts.values());
-  }
-
   /**
    * Step Context for the {@link DirectRunner}.
    */

From 62115b29a7f27a1a74b7c870d4277655adb3dfbf Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 17:46:58 -0700
Subject: [PATCH 299/387] Delete unused BaseStepContext

---
 .../beam/runners/core/BaseStepContext.java    | 40 -------------------
 .../runners/core/SimpleDoFnRunnerTest.java    |  2 +-
 .../runners/core/StatefulDoFnRunnerTest.java  |  2 +-
 .../direct/DirectExecutionContext.java        |  3 --
 4 files changed, 2 insertions(+), 45 deletions(-)
 delete mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
deleted file mode 100644
index 4abd4d228160..000000000000
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseStepContext.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.core;
-
-/**
- * Base class for implementations of {@link StepContext}.
- *
- * <p>To complete a concrete subclass, implement {@link #timerInternals} and
- * {@link #stateInternals}.
- */
-public abstract class BaseStepContext implements StepContext {
-  private final String stepName;
-  private final String transformName;
-
-  public BaseStepContext(String stepName, String transformName) {
-    this.stepName = stepName;
-    this.transformName = transformName;
-  }
-
-  @Override
-  public abstract StateInternals stateInternals();
-
-  @Override
-  public abstract TimerInternals timerInternals();
-}
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index 59e5857c3aa1..f331b65abcb8 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -63,7 +63,7 @@ public class SimpleDoFnRunnerTest {
   @Rule public ExpectedException thrown = ExpectedException.none();
 
   @Mock
-  BaseStepContext mockStepContext;
+  StepContext mockStepContext;
 
   @Mock TimerInternals mockTimerInternals;
 
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
index 62a657882cc6..4f155dca4c3a 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java
@@ -69,7 +69,7 @@ public class StatefulDoFnRunnerTest {
       new IntervalWindow(new Instant(10), new Instant(20));
 
   @Mock
-  BaseStepContext mockStepContext;
+  StepContext mockStepContext;
 
   private InMemoryStateInternals<String> stateInternals;
   private InMemoryTimerInternals timerInternals;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 845256539eb9..cca57193fbd2 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -17,11 +17,8 @@
  */
 package org.apache.beam.runners.direct;
 
-import java.util.Collection;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
-import org.apache.beam.runners.core.BaseStepContext;
 import org.apache.beam.runners.core.StepContext;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;

From 0dc0334a0c1350c1693019f104dac911a618c9c8 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 17:49:01 -0700
Subject: [PATCH 300/387] Shorten excessive name in DirectExecutionContext

---
 .../runners/direct/DirectExecutionContext.java   |  2 +-
 .../GroupAlsoByWindowEvaluatorFactory.java       |  2 +-
 .../runners/direct/ParDoEvaluatorFactory.java    |  2 +-
 ...plittableProcessElementsEvaluatorFactory.java |  2 +-
 .../direct/StatefulParDoEvaluatorFactory.java    |  2 +-
 .../runners/direct/EvaluationContextTest.java    | 16 ++++++++--------
 .../beam/runners/direct/ParDoEvaluatorTest.java  |  2 +-
 .../StatefulParDoEvaluatorFactoryTest.java       |  4 ++--
 8 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index cca57193fbd2..11c1b86a559c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -55,7 +55,7 @@ private DirectStepContext createStepContext(String stepName, String transformNam
   /**
    * Returns the {@link StepContext} associated with the given step.
    */
-  public DirectStepContext getOrCreateStepContext(String stepName, String transformName) {
+  public DirectStepContext getStepContext(String stepName, String transformName) {
     final String finalStepName = stepName;
     final String finalTransformName = transformName;
     DirectStepContext context = cachedStepContexts.get(stepName);
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 78ef7fe5d6da..49b75129b45c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -129,7 +129,7 @@ public GroupAlsoByWindowEvaluator(
       structuralKey = inputBundle.getKey();
       stepContext = evaluationContext
           .getExecutionContext(application, inputBundle.getKey())
-          .getOrCreateStepContext(
+          .getStepContext(
               evaluationContext.getStepName(application), application.getTransform().getName());
       windowingStrategy =
           (WindowingStrategy<?, BoundedWindow>)
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 74470bfb8b8d..12c6751f8e7e 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -112,7 +112,7 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
     DirectStepContext stepContext =
         evaluationContext
             .getExecutionContext(application, inputBundleKey)
-            .getOrCreateStepContext(stepName, stepName);
+            .getStepContext(stepName, stepName);
 
     DoFnLifecycleManager fnManager = fnClones.getUnchecked(doFn);
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index dc85d87bc93f..13d9345094fe 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -109,7 +109,7 @@ public void cleanup() throws Exception {
     final DirectExecutionContext.DirectStepContext stepContext =
         evaluationContext
             .getExecutionContext(application, inputBundle.getKey())
-            .getOrCreateStepContext(stepName, stepName);
+            .getStepContext(stepName, stepName);
 
     final ParDoEvaluator<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
         parDoEvaluator =
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 985c3be4e9e9..70d0cf5bf8f1 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -163,7 +163,7 @@ public Runnable load(
           evaluationContext
               .getExecutionContext(
                   transformOutputWindow.getTransform(), transformOutputWindow.getKey())
-              .getOrCreateStepContext(stepName, stepName);
+              .getStepContext(stepName, stepName);
 
       final StateNamespace namespace =
           StateNamespaces.window(
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 72b1bbcba188..0e2be8d4d5b7 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -160,7 +160,7 @@ public void getExecutionContextSameStepSameKeyState() {
 
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
-    DirectStepContext stepContext = fooContext.getOrCreateStepContext("s1", "s1");
+    DirectStepContext stepContext = fooContext.getStepContext("s1", "s1");
     stepContext.stateInternals().state(StateNamespaces.global(), intBag).add(1);
 
     context.handleResult(
@@ -177,7 +177,7 @@ public void getExecutionContextSameStepSameKeyState() {
             StructuralKey.of("foo", StringUtf8Coder.of()));
     assertThat(
         secondFooContext
-            .getOrCreateStepContext("s1", "s1")
+            .getStepContext("s1", "s1")
             .stateInternals()
             .state(StateNamespaces.global(), intBag)
             .read(),
@@ -194,7 +194,7 @@ public void getExecutionContextDifferentKeysIndependentState() {
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
     fooContext
-        .getOrCreateStepContext("s1", "s1")
+        .getStepContext("s1", "s1")
         .stateInternals()
         .state(StateNamespaces.global(), intBag)
         .add(1);
@@ -205,7 +205,7 @@ public void getExecutionContextDifferentKeysIndependentState() {
     assertThat(barContext, not(equalTo(fooContext)));
     assertThat(
         barContext
-            .getOrCreateStepContext("s1", "s1")
+            .getStepContext("s1", "s1")
             .stateInternals()
             .state(StateNamespaces.global(), intBag)
             .read(),
@@ -221,7 +221,7 @@ public void getExecutionContextDifferentStepsIndependentState() {
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
     fooContext
-        .getOrCreateStepContext("s1", "s1")
+        .getStepContext("s1", "s1")
         .stateInternals()
         .state(StateNamespaces.global(), intBag)
         .add(1);
@@ -230,7 +230,7 @@ public void getExecutionContextDifferentStepsIndependentState() {
         context.getExecutionContext(downstreamProducer, myKey);
     assertThat(
         barContext
-            .getOrCreateStepContext("s1", "s1")
+            .getStepContext("s1", "s1")
             .stateInternals()
             .state(StateNamespaces.global(), intBag)
             .read(),
@@ -246,7 +246,7 @@ public void handleResultStoresState() {
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
     CopyOnAccessInMemoryStateInternals<?> state =
-        fooContext.getOrCreateStepContext("s1", "s1").stateInternals();
+        fooContext.getStepContext("s1", "s1").stateInternals();
     BagState<Integer> bag = state.state(StateNamespaces.global(), intBag);
     bag.add(1);
     bag.add(2);
@@ -266,7 +266,7 @@ public void handleResultStoresState() {
         context.getExecutionContext(downstreamProducer, myKey);
 
     CopyOnAccessInMemoryStateInternals<?> afterResultState =
-        afterResultContext.getOrCreateStepContext("s1", "s1").stateInternals();
+        afterResultContext.getStepContext("s1", "s1").stateInternals();
     assertThat(afterResultState.state(StateNamespaces.global(), intBag).read(), contains(1, 2, 4));
   }
 
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 286e44d1be04..22b3b7e49096 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -140,7 +140,7 @@ private ParDoEvaluator<Integer> createEvaluator(
     DirectExecutionContext executionContext = mock(DirectExecutionContext.class);
     DirectStepContext stepContext = mock(DirectStepContext.class);
     when(
-            executionContext.getOrCreateStepContext(
+            executionContext.getStepContext(
                 Mockito.any(String.class), Mockito.any(String.class)))
         .thenReturn(stepContext);
     when(stepContext.getTimerUpdate()).thenReturn(TimerUpdate.empty());
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
index eb54d5c818dc..b233c1bfe868 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
@@ -153,7 +153,7 @@ public void process(ProcessContext c) {}
     when(mockEvaluationContext.getExecutionContext(
             eq(producingTransform), Mockito.<StructuralKey>any()))
         .thenReturn(mockExecutionContext);
-    when(mockExecutionContext.getOrCreateStepContext(anyString(), anyString()))
+    when(mockExecutionContext.getStepContext(anyString(), anyString()))
         .thenReturn(mockStepContext);
 
     IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(9));
@@ -269,7 +269,7 @@ public void process(ProcessContext c) {}
     when(mockEvaluationContext.getExecutionContext(
             eq(producingTransform), Mockito.<StructuralKey>any()))
         .thenReturn(mockExecutionContext);
-    when(mockExecutionContext.getOrCreateStepContext(anyString(), anyString()))
+    when(mockExecutionContext.getStepContext(anyString(), anyString()))
         .thenReturn(mockStepContext);
     when(mockEvaluationContext.createBundle(Matchers.<PCollection<Integer>>any()))
         .thenReturn(mockUncommittedBundle);

From d425b2792f754ed6150f7b47eddf743286a45401 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 11:10:42 -0700
Subject: [PATCH 301/387] Revise StepContext javadoc

---
 .../java/org/apache/beam/runners/core/StepContext.java     | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
index 60fc40291452..4d66d668c7ff 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StepContext.java
@@ -18,7 +18,12 @@
 package org.apache.beam.runners.core;
 
 /**
- * Per-step, per-key context used for retrieving state.
+ * The context in which a specific step is executing, including access to state and timers.
+ *
+ * <p>This interface exists as the API between a runner and the support code, but is not user
+ * facing.
+ *
+ * <p>These will often be scoped to a particular step and key, though it is not required.
  */
 public interface StepContext {
 

From b32a1c350398a91b1b1552d5257dab6ab7d1da3a Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 11:13:19 -0700
Subject: [PATCH 302/387] Remove unused pieces of DirectStepContext

---
 .../runners/direct/DirectExecutionContext.java | 18 +++++-------------
 .../GroupAlsoByWindowEvaluatorFactory.java     |  2 +-
 .../runners/direct/ParDoEvaluatorFactory.java  |  2 +-
 ...ittableProcessElementsEvaluatorFactory.java |  2 +-
 .../direct/StatefulParDoEvaluatorFactory.java  |  2 +-
 .../runners/direct/EvaluationContextTest.java  | 16 ++++++++--------
 .../runners/direct/ParDoEvaluatorTest.java     |  2 +-
 .../StatefulParDoEvaluatorFactoryTest.java     |  4 ++--
 8 files changed, 20 insertions(+), 28 deletions(-)

diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
index 11c1b86a559c..e8ad8d75f351 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -48,19 +48,17 @@ public DirectExecutionContext(
     this.watermarks = watermarks;
   }
 
-  private DirectStepContext createStepContext(String stepName, String transformName) {
-    return new DirectStepContext(stepName, transformName);
+  private DirectStepContext createStepContext() {
+    return new DirectStepContext();
   }
 
   /**
    * Returns the {@link StepContext} associated with the given step.
    */
-  public DirectStepContext getStepContext(String stepName, String transformName) {
-    final String finalStepName = stepName;
-    final String finalTransformName = transformName;
+  public DirectStepContext getStepContext(String stepName) {
     DirectStepContext context = cachedStepContexts.get(stepName);
     if (context == null) {
-      context = createStepContext(finalStepName, finalTransformName);
+      context = createStepContext();
       cachedStepContexts.put(stepName, context);
     }
     return context;
@@ -72,14 +70,8 @@ public DirectStepContext getStepContext(String stepName, String transformName) {
   public class DirectStepContext implements StepContext {
     private CopyOnAccessInMemoryStateInternals<?> stateInternals;
     private DirectTimerInternals timerInternals;
-    private final String stepName;
-    private final String transformName;
 
-    public DirectStepContext(
-        String stepName, String transformName) {
-      this.stepName = stepName;
-      this.transformName = transformName;
-    }
+    public DirectStepContext() { }
 
     @Override
     public CopyOnAccessInMemoryStateInternals<?> stateInternals() {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 49b75129b45c..1a588eedf895 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -130,7 +130,7 @@ public GroupAlsoByWindowEvaluator(
       stepContext = evaluationContext
           .getExecutionContext(application, inputBundle.getKey())
           .getStepContext(
-              evaluationContext.getStepName(application), application.getTransform().getName());
+              evaluationContext.getStepName(application));
       windowingStrategy =
           (WindowingStrategy<?, BoundedWindow>)
               application.getTransform().getInputWindowingStrategy();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 12c6751f8e7e..8aa75cf1445e 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -112,7 +112,7 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
     DirectStepContext stepContext =
         evaluationContext
             .getExecutionContext(application, inputBundleKey)
-            .getStepContext(stepName, stepName);
+            .getStepContext(stepName);
 
     DoFnLifecycleManager fnManager = fnClones.getUnchecked(doFn);
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index 13d9345094fe..b85f481c1489 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -109,7 +109,7 @@ public void cleanup() throws Exception {
     final DirectExecutionContext.DirectStepContext stepContext =
         evaluationContext
             .getExecutionContext(application, inputBundle.getKey())
-            .getStepContext(stepName, stepName);
+            .getStepContext(stepName);
 
     final ParDoEvaluator<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>
         parDoEvaluator =
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 70d0cf5bf8f1..506c84cec639 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -163,7 +163,7 @@ public Runnable load(
           evaluationContext
               .getExecutionContext(
                   transformOutputWindow.getTransform(), transformOutputWindow.getKey())
-              .getStepContext(stepName, stepName);
+              .getStepContext(stepName);
 
       final StateNamespace namespace =
           StateNamespaces.window(
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index 0e2be8d4d5b7..80b04f80a06a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -160,7 +160,7 @@ public void getExecutionContextSameStepSameKeyState() {
 
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
-    DirectStepContext stepContext = fooContext.getStepContext("s1", "s1");
+    DirectStepContext stepContext = fooContext.getStepContext("s1");
     stepContext.stateInternals().state(StateNamespaces.global(), intBag).add(1);
 
     context.handleResult(
@@ -177,7 +177,7 @@ public void getExecutionContextSameStepSameKeyState() {
             StructuralKey.of("foo", StringUtf8Coder.of()));
     assertThat(
         secondFooContext
-            .getStepContext("s1", "s1")
+            .getStepContext("s1")
             .stateInternals()
             .state(StateNamespaces.global(), intBag)
             .read(),
@@ -194,7 +194,7 @@ public void getExecutionContextDifferentKeysIndependentState() {
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
     fooContext
-        .getStepContext("s1", "s1")
+        .getStepContext("s1")
         .stateInternals()
         .state(StateNamespaces.global(), intBag)
         .add(1);
@@ -205,7 +205,7 @@ public void getExecutionContextDifferentKeysIndependentState() {
     assertThat(barContext, not(equalTo(fooContext)));
     assertThat(
         barContext
-            .getStepContext("s1", "s1")
+            .getStepContext("s1")
             .stateInternals()
             .state(StateNamespaces.global(), intBag)
             .read(),
@@ -221,7 +221,7 @@ public void getExecutionContextDifferentStepsIndependentState() {
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
     fooContext
-        .getStepContext("s1", "s1")
+        .getStepContext("s1")
         .stateInternals()
         .state(StateNamespaces.global(), intBag)
         .add(1);
@@ -230,7 +230,7 @@ public void getExecutionContextDifferentStepsIndependentState() {
         context.getExecutionContext(downstreamProducer, myKey);
     assertThat(
         barContext
-            .getStepContext("s1", "s1")
+            .getStepContext("s1")
             .stateInternals()
             .state(StateNamespaces.global(), intBag)
             .read(),
@@ -246,7 +246,7 @@ public void handleResultStoresState() {
     StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
 
     CopyOnAccessInMemoryStateInternals<?> state =
-        fooContext.getStepContext("s1", "s1").stateInternals();
+        fooContext.getStepContext("s1").stateInternals();
     BagState<Integer> bag = state.state(StateNamespaces.global(), intBag);
     bag.add(1);
     bag.add(2);
@@ -266,7 +266,7 @@ public void handleResultStoresState() {
         context.getExecutionContext(downstreamProducer, myKey);
 
     CopyOnAccessInMemoryStateInternals<?> afterResultState =
-        afterResultContext.getStepContext("s1", "s1").stateInternals();
+        afterResultContext.getStepContext("s1").stateInternals();
     assertThat(afterResultState.state(StateNamespaces.global(), intBag).read(), contains(1, 2, 4));
   }
 
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 22b3b7e49096..09a21ac524a5 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -141,7 +141,7 @@ private ParDoEvaluator<Integer> createEvaluator(
     DirectStepContext stepContext = mock(DirectStepContext.class);
     when(
             executionContext.getStepContext(
-                Mockito.any(String.class), Mockito.any(String.class)))
+                Mockito.any(String.class)))
         .thenReturn(stepContext);
     when(stepContext.getTimerUpdate()).thenReturn(TimerUpdate.empty());
     when(
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
index b233c1bfe868..9366b7c9ff8c 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java
@@ -153,7 +153,7 @@ public void process(ProcessContext c) {}
     when(mockEvaluationContext.getExecutionContext(
             eq(producingTransform), Mockito.<StructuralKey>any()))
         .thenReturn(mockExecutionContext);
-    when(mockExecutionContext.getStepContext(anyString(), anyString()))
+    when(mockExecutionContext.getStepContext(anyString()))
         .thenReturn(mockStepContext);
 
     IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(9));
@@ -269,7 +269,7 @@ public void process(ProcessContext c) {}
     when(mockEvaluationContext.getExecutionContext(
             eq(producingTransform), Mockito.<StructuralKey>any()))
         .thenReturn(mockExecutionContext);
-    when(mockExecutionContext.getStepContext(anyString(), anyString()))
+    when(mockExecutionContext.getStepContext(anyString()))
         .thenReturn(mockStepContext);
     when(mockEvaluationContext.createBundle(Matchers.<PCollection<Integer>>any()))
         .thenReturn(mockUncommittedBundle);

From 329e4bd3bfa3414b661c8253f612f48527c783c0 Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Tue, 23 May 2017 10:42:54 -0700
Subject: [PATCH 303/387] Fix lint error in datastoreio_test

---
 sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
index 424e714c5737..94cac3e58dcc 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 
-import math
 import unittest
 
 from mock import MagicMock, call, patch

From 2e86f4faf9cba1fc15969f3c1f3c3463a332f7ca Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 23 May 2017 13:24:27 -0700
Subject: [PATCH 304/387] [BEAM-2338] Fix the limit counter in gcsio reads

---
 sdks/python/apache_beam/io/gcp/gcsio.py      |  2 +-
 sdks/python/apache_beam/io/gcp/gcsio_test.py | 22 +++++++++++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index 7e21586381c2..d43c8ba0d972 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -392,7 +392,7 @@ def size_of_files_in_glob(self, pattern, limit=None):
         if fnmatch.fnmatch(item.name, name_pattern):
           file_name = 'gs://%s/%s' % (item.bucket, item.name)
           file_sizes[file_name] = item.size
-        counter += 1
+          counter += 1
         if limit is not None and counter >= limit:
           break
         if counter % 10000 == 0:
diff --git a/sdks/python/apache_beam/io/gcp/gcsio_test.py b/sdks/python/apache_beam/io/gcp/gcsio_test.py
index 73d221376a5d..06a82272900e 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio_test.py
@@ -641,6 +641,7 @@ def test_glob(self):
         'apple/fish/cat',
         'apple/fish/cart',
         'apple/fish/carl',
+        'apple/fish/handle',
         'apple/dish/bat',
         'apple/dish/cat',
         'apple/dish/carl',
@@ -661,6 +662,7 @@ def test_glob(self):
             'apple/fish/cat',
             'apple/fish/cart',
             'apple/fish/carl',
+            'apple/fish/handle',
             'apple/dish/bat',
             'apple/dish/cat',
             'apple/dish/carl',
@@ -691,6 +693,12 @@ def test_glob(self):
             'apple/fish/bambi',
             'apple/fish/balloon',
         ]),
+        ('gs://gcsio-test/apple/f*/b*', [
+            'apple/fish/blubber',
+            'apple/fish/blowfish',
+            'apple/fish/bambi',
+            'apple/fish/balloon',
+        ]),
         ('gs://gcsio-test/apple/dish/[cb]at', [
             'apple/dish/bat',
             'apple/dish/cat',
@@ -726,6 +734,7 @@ def test_size_of_files_in_glob(self):
         ('apple/dish/bat', 13),
         ('apple/dish/cat', 14),
         ('apple/dish/carl', 15),
+        ('apple/fish/handle', 16),
     ]
     for (object_name, size) in object_names:
       file_name = 'gs://%s/%s' % (bucket_name, object_name)
@@ -739,7 +748,11 @@ def test_size_of_files_in_glob(self):
         ('gs://gcsio-test/apple/fish/car?', [
             ('apple/fish/cart', 11),
             ('apple/fish/carl', 12),
-        ])
+        ]),
+        ('gs://gcsio-test/*/f*/car?', [
+            ('apple/fish/cart', 11),
+            ('apple/fish/carl', 12),
+        ]),
     ]
     for file_pattern, expected_object_names in test_cases:
       expected_file_sizes = {'gs://%s/%s' % (bucket_name, o): s
@@ -747,6 +760,13 @@ def test_size_of_files_in_glob(self):
       self.assertEqual(
           self.gcs.size_of_files_in_glob(file_pattern), expected_file_sizes)
 
+    # Check if limits are followed correctly
+    limit = 1
+    for file_pattern, expected_object_names in test_cases:
+      expected_num_items = min(len(expected_object_names), limit)
+      self.assertEqual(
+          len(self.gcs.glob(file_pattern, limit)), expected_num_items)
+
   def test_size_of_files_in_glob_limited(self):
     bucket_name = 'gcsio-test'
     object_names = [

From 940819e2acccb8d77b88aaee821ee972aca02eb3 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:26:44 -0700
Subject: [PATCH 305/387] Rename PCollections to PCollectionTranslation

This is to give a standard and obvious suffix for all our helper classes
for translating to/from Runner API protos.
---
 ...lections.java => PCollectionTranslation.java} |  4 ++--
 .../runners/core/construction/SdkComponents.java |  3 ++-
 ...Test.java => PCollectionTranslationTest.java} | 16 +++++++++-------
 3 files changed, 13 insertions(+), 10 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{PCollections.java => PCollectionTranslation.java} (98%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{PCollectionsTest.java => PCollectionTranslationTest.java} (92%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
similarity index 98%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
index 0f2fcb7ca281..cad7b97c74b5 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollections.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
@@ -29,8 +29,8 @@
 /**
  * Utility methods for translating {@link PCollection PCollections} to and from Runner API protos.
  */
-public class PCollections {
-  private PCollections() {}
+public class PCollectionTranslation {
+  private PCollectionTranslation() {}
 
   public static RunnerApi.PCollection toProto(PCollection<?> pCollection, SdkComponents components)
       throws IOException {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index 5714fc510481..3d8d4cd3b94e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -176,7 +176,8 @@ String registerPCollection(PCollection<?> pCollection) throws IOException {
     }
     String uniqueName = uniqify(pCollection.getName(), pCollectionIds.values());
     pCollectionIds.put(pCollection, uniqueName);
-    componentsBuilder.putPcollections(uniqueName, PCollections.toProto(pCollection, this));
+    componentsBuilder.putPcollections(
+        uniqueName, PCollectionTranslation.toProto(pCollection, this));
     return uniqueName;
   }
 
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
similarity index 92%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
index 9407a5a5cda2..3b942206b112 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java
@@ -64,10 +64,10 @@
 import org.junit.runners.Parameterized.Parameters;
 
 /**
- * Tests for {@link PCollections}.
+ * Tests for {@link PCollectionTranslation}.
  */
 @RunWith(Parameterized.class)
-public class PCollectionsTest {
+public class PCollectionTranslationTest {
   // Each spec activates tests of all subsets of its fields
   @Parameters(name = "{index}: {0}")
   public static Iterable<PCollection<?>> data() {
@@ -91,7 +91,8 @@ public static Iterable<PCollection<?>> data() {
         pipeline
             .apply(
                 "intsWithCustomCoder",
-                Create.of(1, 2).withCoder(new AutoValue_PCollectionsTest_CustomIntCoder()))
+                Create.of(1, 2)
+                    .withCoder(new AutoValue_PCollectionTranslationTest_CustomIntCoder()))
             .apply(
                 "into custom windows",
                 Window.<Integer>into(new CustomWindows())
@@ -113,12 +114,13 @@ public static Iterable<PCollection<?>> data() {
   @Test
   public void testEncodeDecodeCycle() throws Exception {
     SdkComponents sdkComponents = SdkComponents.create();
-    RunnerApi.PCollection protoCollection = PCollections.toProto(testCollection, sdkComponents);
+    RunnerApi.PCollection protoCollection = PCollectionTranslation
+        .toProto(testCollection, sdkComponents);
     RunnerApi.Components protoComponents = sdkComponents.toComponents();
-    Coder<?> decodedCoder = PCollections.getCoder(protoCollection, protoComponents);
+    Coder<?> decodedCoder = PCollectionTranslation.getCoder(protoCollection, protoComponents);
     WindowingStrategy<?, ?> decodedStrategy =
-        PCollections.getWindowingStrategy(protoCollection, protoComponents);
-    IsBounded decodedIsBounded = PCollections.isBounded(protoCollection);
+        PCollectionTranslation.getWindowingStrategy(protoCollection, protoComponents);
+    IsBounded decodedIsBounded = PCollectionTranslation.isBounded(protoCollection);
 
     assertThat(decodedCoder, Matchers.<Coder<?>>equalTo(testCollection.getCoder()));
     assertThat(

From 9b6728e24748791b7181b20183df3ada31f45682 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:28:08 -0700
Subject: [PATCH 306/387] Rename PTransforms to PTransformTranslation

---
 .../{PTransforms.java => PTransformTranslation.java}  |  4 ++--
 .../apache/beam/runners/core/construction/ParDos.java |  4 ++--
 .../beam/runners/core/construction/SdkComponents.java |  3 ++-
 .../TransformPayloadTranslatorRegistrar.java          |  2 +-
 .../core/construction/WindowIntoTranslator.java       |  2 +-
 ...sformsTest.java => PTransformTranslationTest.java} | 11 ++++++-----
 6 files changed, 14 insertions(+), 12 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{PTransforms.java => PTransformTranslation.java} (98%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{PTransformsTest.java => PTransformTranslationTest.java} (95%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
similarity index 98%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 9826b77b300a..86638dec8098 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransforms.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -37,7 +37,7 @@
  * Utilities for converting {@link PTransform PTransforms} to and from {@link RunnerApi Runner API
  * protocol buffers}.
  */
-public class PTransforms {
+public class PTransformTranslation {
   private static final Map<Class<? extends PTransform>, TransformPayloadTranslator>
       KNOWN_PAYLOAD_TRANSLATORS = loadTransformPayloadTranslators();
 
@@ -52,7 +52,7 @@ public class PTransforms {
     return builder.build();
   }
 
-  private PTransforms() {}
+  private PTransformTranslation() {}
 
   /**
    * Translates an {@link AppliedPTransform} into a runner API proto.
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
index 2ecc04161fed..12f29694ca1e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
@@ -34,7 +34,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import org.apache.beam.runners.core.construction.PTransforms.TransformPayloadTranslator;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
@@ -96,7 +96,7 @@ public class ParDos {
    * A {@link TransformPayloadTranslator} for {@link ParDo}.
    */
   public static class ParDoPayloadTranslator
-      implements PTransforms.TransformPayloadTranslator<ParDo.MultiOutput<?, ?>> {
+      implements PTransformTranslation.TransformPayloadTranslator<ParDo.MultiOutput<?, ?>> {
     public static TransformPayloadTranslator create() {
       return new ParDoPayloadTranslator();
     }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index 3d8d4cd3b94e..da229820a984 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -131,7 +131,8 @@ String registerPTransform(
       return name;
     }
     checkNotNull(children, "child nodes may not be null");
-    componentsBuilder.putTransforms(name, PTransforms.toProto(appliedPTransform, children, this));
+    componentsBuilder.putTransforms(name, PTransformTranslation
+        .toProto(appliedPTransform, children, this));
     return name;
   }
 
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
index bc568a616159..3b3ffa18b26b 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformPayloadTranslatorRegistrar.java
@@ -19,7 +19,7 @@
 package org.apache.beam.runners.core.construction;
 
 import java.util.Map;
-import org.apache.beam.runners.core.construction.PTransforms.TransformPayloadTranslator;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /** A registrar of TransformPayloadTranslator. */
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
index ea4c9965790c..7ed2a499a95e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
@@ -20,7 +20,7 @@
 
 import com.google.protobuf.Any;
 import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.beam.runners.core.construction.PTransforms.TransformPayloadTranslator;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
similarity index 95%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
index 41255448dee7..0e6ef975c6c8 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java
@@ -59,10 +59,10 @@
 import org.junit.runners.Parameterized.Parameters;
 
 /**
- * Tests for {@link PTransforms}.
+ * Tests for {@link PTransformTranslation}.
  */
 @RunWith(Parameterized.class)
-public class PTransformsTest {
+public class PTransformTranslationTest {
 
   @Parameters(name = "{index}: {0}")
   public static Iterable<ToAndFromProtoSpec> data() {
@@ -88,7 +88,7 @@ public static Iterable<ToAndFromProtoSpec> data() {
   @AutoValue
   abstract static class ToAndFromProtoSpec {
     public static ToAndFromProtoSpec leaf(AppliedPTransform<?, ?, ?> transform) {
-      return new AutoValue_PTransformsTest_ToAndFromProtoSpec(
+      return new AutoValue_PTransformTranslationTest_ToAndFromProtoSpec(
           transform, Collections.<ToAndFromProtoSpec>emptyList());
     }
 
@@ -97,7 +97,7 @@ public static ToAndFromProtoSpec composite(
       List<ToAndFromProtoSpec> childSpecs = new ArrayList<>();
       childSpecs.add(spec);
       childSpecs.addAll(Arrays.asList(specs));
-      return new AutoValue_PTransformsTest_ToAndFromProtoSpec(topLevel, childSpecs);
+      return new AutoValue_PTransformTranslationTest_ToAndFromProtoSpec(topLevel, childSpecs);
     }
 
     abstract AppliedPTransform<?, ?, ?> getTransform();
@@ -139,7 +139,8 @@ private RunnerApi.PTransform convert(ToAndFromProtoSpec spec, SdkComponents comp
       // Sanity call
       components.getExistingPTransformId(child.getTransform());
     }
-    PTransform convert = PTransforms.toProto(spec.getTransform(), childTransforms, components);
+    PTransform convert = PTransformTranslation
+        .toProto(spec.getTransform(), childTransforms, components);
     // Make sure the converted transform is registered. Convert it independently, but if this is a
     // child spec, the child must be in the components.
     components.registerPTransform(spec.getTransform(), childTransforms);

From 446093836016dabf021d34ca0a858e313f493e2f Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:28:49 -0700
Subject: [PATCH 307/387] Rename ParDos to ParDoTranslation

---
 .../{ParDos.java => ParDoTranslation.java}        |  4 ++--
 ...{ParDosTest.java => ParDoTranslationTest.java} | 15 ++++++++-------
 2 files changed, 10 insertions(+), 9 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{ParDos.java => ParDoTranslation.java} (99%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{ParDosTest.java => ParDoTranslationTest.java} (95%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
similarity index 99%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
index 12f29694ca1e..baed246c7cb8 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDos.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -73,7 +73,7 @@
 /**
  * Utilities for interacting with {@link ParDo} instances and {@link ParDoPayload} protos.
  */
-public class ParDos {
+public class ParDoTranslation {
   /**
    * The URN for a {@link ParDoPayload}.
    */
@@ -191,7 +191,7 @@ private static TimerSpec toProto(TimerDeclaration timer) {
   abstract static class DoFnAndMainOutput implements Serializable {
     public static DoFnAndMainOutput of(
         DoFn<?, ?> fn, TupleTag<?> tag) {
-      return new AutoValue_ParDos_DoFnAndMainOutput(fn, tag);
+      return new AutoValue_ParDoTranslation_DoFnAndMainOutput(fn, tag);
     }
 
     abstract DoFn<?, ?> getDoFn();
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
similarity index 95%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
index b6f0b7d9d68e..ec2795746451 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDosTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
@@ -66,9 +66,9 @@
 import org.junit.runners.Parameterized.Parameter;
 import org.junit.runners.Parameterized.Parameters;
 
-/** Tests for {@link ParDos}. */
+/** Tests for {@link ParDoTranslation}. */
 @RunWith(Parameterized.class)
-public class ParDosTest {
+public class ParDoTranslationTest {
   public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
 
   private static PCollectionView<Long> singletonSideInput =
@@ -106,11 +106,12 @@ public class ParDosTest {
   @Test
   public void testToAndFromProto() throws Exception {
     SdkComponents components = SdkComponents.create();
-    ParDoPayload payload = ParDos.toProto(parDo, components);
+    ParDoPayload payload = ParDoTranslation.toProto(parDo, components);
 
-    assertThat(ParDos.getDoFn(payload), Matchers.<DoFn<?, ?>>equalTo(parDo.getFn()));
+    assertThat(ParDoTranslation.getDoFn(payload), Matchers.<DoFn<?, ?>>equalTo(parDo.getFn()));
     assertThat(
-        ParDos.getMainOutputTag(payload), Matchers.<TupleTag<?>>equalTo(parDo.getMainOutputTag()));
+        ParDoTranslation.getMainOutputTag(payload),
+        Matchers.<TupleTag<?>>equalTo(parDo.getMainOutputTag()));
     for (PCollectionView<?> view : parDo.getSideInputs()) {
       payload.getSideInputsOrThrow(view.getTagInternal().getId());
     }
@@ -137,7 +138,7 @@ public void toAndFromTransformProto() throws Exception {
     for (PCollectionView<?> view : parDo.getSideInputs()) {
       SideInput sideInput = parDoPayload.getSideInputsOrThrow(view.getTagInternal().getId());
       PCollectionView<?> restoredView =
-          ParDos.fromProto(
+          ParDoTranslation.fromProto(
               sideInput, view.getTagInternal().getId(), protoTransform, protoComponents);
       assertThat(restoredView.getTagInternal(), equalTo(view.getTagInternal()));
       assertThat(restoredView.getViewFn(), instanceOf(view.getViewFn().getClass()));
@@ -151,7 +152,7 @@ public void toAndFromTransformProto() throws Exception {
     }
     String mainInputId = components.registerPCollection(mainInput);
     assertThat(
-        ParDos.getMainInput(protoTransform, protoComponents),
+        ParDoTranslation.getMainInput(protoTransform, protoComponents),
         equalTo(protoComponents.getPcollectionsOrThrow(mainInputId)));
   }
 

From bc4f44f46ca6afc0018834eb467b1112763c3323 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:29:16 -0700
Subject: [PATCH 308/387] Rename WindowIntoTranslator to WindowIntoTranslation

---
 ...ndowIntoTranslator.java => WindowIntoTranslation.java} | 4 ++--
 ...TranslatorTest.java => WindowIntoTranslationTest.java} | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{WindowIntoTranslator.java => WindowIntoTranslation.java} (94%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{WindowIntoTranslatorTest.java => WindowIntoTranslationTest.java} (95%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
similarity index 94%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
index 7ed2a499a95e..69793b56d83c 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslator.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
@@ -33,7 +33,7 @@
  * Utility methods for translating a {@link Window.Assign} to and from {@link RunnerApi}
  * representations.
  */
-public class WindowIntoTranslator {
+public class WindowIntoTranslation {
 
   static class WindowAssignTranslator implements TransformPayloadTranslator<Window.Assign<?>> {
     @Override
@@ -42,7 +42,7 @@ public FunctionSpec translate(
       return FunctionSpec.newBuilder()
           .setUrn("urn:beam:transform:window:v1")
           .setParameter(
-              Any.pack(WindowIntoTranslator.toProto(transform.getTransform(), components)))
+              Any.pack(WindowIntoTranslation.toProto(transform.getTransform(), components)))
           .build();
     }
   }
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
similarity index 95%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
index eaefe2eade3b..cb9617abd188 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslatorTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java
@@ -51,10 +51,10 @@
 import org.junit.runners.Parameterized.Parameters;
 
 /**
- * Tests for {@link WindowIntoTranslator}.
+ * Tests for {@link WindowIntoTranslation}.
  */
 @RunWith(Parameterized.class)
-public class WindowIntoTranslatorTest {
+public class WindowIntoTranslationTest {
   @Parameters(name = "{index}: {0}")
   public static Iterable<WindowFn<?, ?>> data() {
     // This pipeline exists for construction, not to run any test.
@@ -93,9 +93,9 @@ public void visitPrimitiveTransform(Node node) {
 
     SdkComponents components = SdkComponents.create();
     WindowIntoPayload payload =
-        WindowIntoTranslator.toProto(assign.get().getTransform(), components);
+        WindowIntoTranslation.toProto(assign.get().getTransform(), components);
 
-    assertEquals(windowFn, WindowIntoTranslator.getWindowFn(payload));
+    assertEquals(windowFn, WindowIntoTranslation.getWindowFn(payload));
   }
 
   private static class CustomWindows extends PartitioningWindowFn<String, BoundedWindow> {

From b35e91d4ed99b74d37a08a1385018b4ca326b3a0 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:30:22 -0700
Subject: [PATCH 309/387] Rename ReadTranslator to ReadTranslation

---
 .../{ReadTranslator.java => ReadTranslation.java}    |  2 +-
 ...dTranslatorTest.java => ReadTranslationTest.java} | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{ReadTranslator.java => ReadTranslation.java} (99%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{ReadTranslatorTest.java => ReadTranslationTest.java} (93%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
similarity index 99%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslator.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
index f94493862b2e..d6c34008f29e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslator.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
@@ -39,7 +39,7 @@
  * Methods for translating {@link Read.Bounded} and {@link Read.Unbounded}
  * {@link PTransform PTransforms} into {@link ReadPayload} protos.
  */
-public class ReadTranslator {
+public class ReadTranslation {
   private static final String JAVA_SERIALIZED_BOUNDED_SOURCE = "urn:beam:java:boundedsource:v1";
   private static final String JAVA_SERIALIZED_UNBOUNDED_SOURCE = "urn:beam:java:unboundedsource:v1";
 
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslatorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
similarity index 93%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslatorTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
index a603e342bc9f..740b3245dd6a 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslatorTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java
@@ -51,10 +51,10 @@
 import org.junit.runners.Parameterized.Parameters;
 
 /**
- * Tests for {@link ReadTranslator}.
+ * Tests for {@link ReadTranslation}.
  */
 @RunWith(Parameterized.class)
-public class ReadTranslatorTest {
+public class ReadTranslationTest {
 
   @Parameters(name = "{index}: {0}")
   public static Iterable<Source<?>> data() {
@@ -74,9 +74,9 @@ public void testToFromProtoBounded() throws Exception {
     assumeThat(source, instanceOf(BoundedSource.class));
     BoundedSource<?> boundedSource = (BoundedSource<?>) this.source;
     Read.Bounded<?> boundedRead = Read.from(boundedSource);
-    ReadPayload payload = ReadTranslator.toProto(boundedRead);
+    ReadPayload payload = ReadTranslation.toProto(boundedRead);
     assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.BOUNDED));
-    BoundedSource<?> deserializedSource = ReadTranslator.boundedSourceFromProto(payload);
+    BoundedSource<?> deserializedSource = ReadTranslation.boundedSourceFromProto(payload);
     assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
   }
 
@@ -85,9 +85,9 @@ public void testToFromProtoUnbounded() throws Exception {
     assumeThat(source, instanceOf(UnboundedSource.class));
     UnboundedSource<?, ?> unboundedSource = (UnboundedSource<?, ?>) this.source;
     Read.Unbounded<?> unboundedRead = Read.from(unboundedSource);
-    ReadPayload payload = ReadTranslator.toProto(unboundedRead);
+    ReadPayload payload = ReadTranslation.toProto(unboundedRead);
     assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.UNBOUNDED));
-    UnboundedSource<?, ?> deserializedSource = ReadTranslator.unboundedSourceFromProto(payload);
+    UnboundedSource<?, ?> deserializedSource = ReadTranslation.unboundedSourceFromProto(payload);
     assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
   }
 

From 7e37b70317bd06f300a5423cc4cb76a06c3955c3 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:31:49 -0700
Subject: [PATCH 310/387] Rename Coders to CoderTranslation

---
 .../{Coders.java => CoderTranslation.java}    |  2 +-
 .../construction/PCollectionTranslation.java  |  3 ++-
 .../core/construction/ParDoTranslation.java   |  3 ++-
 .../core/construction/SdkComponents.java      |  2 +-
 ...ersTest.java => CoderTranslationTest.java} | 19 ++++++++++---------
 5 files changed, 16 insertions(+), 13 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{Coders.java => CoderTranslation.java} (99%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{CodersTest.java => CoderTranslationTest.java} (89%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Coders.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
similarity index 99%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Coders.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
index 6c2caa9ba0c0..470db6ab8c75 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Coders.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java
@@ -49,7 +49,7 @@
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
 
 /** Converts to and from Beam Runner API representations of {@link Coder Coders}. */
-public class Coders {
+public class CoderTranslation {
   // This URN says that the coder is just a UDF blob this SDK understands
   // TODO: standardize such things
   public static final String JAVA_SERIALIZED_CODER_URN = "urn:beam:coders:javasdk:0.1";
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
index cad7b97c74b5..46f714e66ef5 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
@@ -53,7 +53,8 @@ public static IsBounded isBounded(RunnerApi.PCollection pCollection) {
 
   public static Coder<?> getCoder(
       RunnerApi.PCollection pCollection, RunnerApi.Components components) throws IOException {
-    return Coders.fromProto(components.getCodersOrThrow(pCollection.getCoderId()), components);
+    return CoderTranslation
+        .fromProto(components.getCodersOrThrow(pCollection.getCoderId()), components);
   }
 
   public static WindowingStrategy<?, ?> getWindowingStrategy(
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
index baed246c7cb8..bc5bb0e414a3 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -269,7 +269,8 @@ public static PCollectionView<?> fromProto(
             components.getWindowingStrategiesOrThrow(inputCollection.getWindowingStrategyId()),
             components);
     Coder<?> elemCoder =
-        Coders.fromProto(components.getCodersOrThrow(inputCollection.getCoderId()), components);
+        CoderTranslation
+            .fromProto(components.getCodersOrThrow(inputCollection.getCoderId()), components);
     Coder<Iterable<WindowedValue<?>>> coder =
         (Coder)
             IterableCoder.of(
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index da229820a984..5c8187513402 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -222,7 +222,7 @@ String registerCoder(Coder<?> coder) throws IOException {
     String baseName = NameUtils.approximateSimpleName(coder);
     String name = uniqify(baseName, coderIds.values());
     coderIds.put(Equivalence.identity().wrap(coder), name);
-    RunnerApi.Coder coderProto = Coders.toProto(coder, this);
+    RunnerApi.Coder coderProto = CoderTranslation.toProto(coder, this);
     componentsBuilder.putCoders(name, coderProto);
     return name;
   }
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
similarity index 89%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
index 42fba7cda508..39549d062ed6 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CodersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java
@@ -56,9 +56,9 @@
 import org.junit.runners.Parameterized.Parameter;
 import org.junit.runners.Parameterized.Parameters;
 
-/** Tests for {@link Coders}. */
+/** Tests for {@link CoderTranslation}. */
 @RunWith(Enclosed.class)
-public class CodersTest {
+public class CoderTranslationTest {
   private static final Set<StructuredCoder<?>> KNOWN_CODERS =
       ImmutableSet.<StructuredCoder<?>>builder()
           .add(ByteArrayCoder.of())
@@ -84,7 +84,8 @@ public void validateKnownCoders() {
       // Validates that every known coder in the Coders class is represented in a "Known Coder"
       // tests, which demonstrates that they are serialized via components and specified URNs rather
       // than java serialized
-      Set<Class<? extends StructuredCoder>> knownCoderClasses = Coders.KNOWN_CODER_URNS.keySet();
+      Set<Class<? extends StructuredCoder>> knownCoderClasses =
+          CoderTranslation.KNOWN_CODER_URNS.keySet();
       Set<Class<? extends StructuredCoder>> knownCoderTests = new HashSet<>();
       for (StructuredCoder<?> coder : KNOWN_CODERS) {
         knownCoderTests.add(coder.getClass());
@@ -94,7 +95,7 @@ public void validateKnownCoders() {
       assertThat(
           String.format(
               "Missing validation of known coder %s in %s",
-              missingKnownCoders, CodersTest.class.getSimpleName()),
+              missingKnownCoders, CoderTranslationTest.class.getSimpleName()),
           missingKnownCoders,
           Matchers.empty());
     }
@@ -103,8 +104,8 @@ public void validateKnownCoders() {
     public void validateCoderTranslators() {
       assertThat(
           "Every Known Coder must have a Known Translator",
-          Coders.KNOWN_CODER_URNS.keySet(),
-          equalTo(Coders.KNOWN_TRANSLATORS.keySet()));
+          CoderTranslation.KNOWN_CODER_URNS.keySet(),
+          equalTo(CoderTranslation.KNOWN_TRANSLATORS.keySet()));
     }
   }
 
@@ -132,17 +133,17 @@ public static Iterable<Coder<?>> data() {
     @Test
     public void toAndFromProto() throws Exception {
       SdkComponents componentsBuilder = SdkComponents.create();
-      RunnerApi.Coder coderProto = Coders.toProto(coder, componentsBuilder);
+      RunnerApi.Coder coderProto = CoderTranslation.toProto(coder, componentsBuilder);
 
       Components encodedComponents = componentsBuilder.toComponents();
-      Coder<?> decodedCoder = Coders.fromProto(coderProto, encodedComponents);
+      Coder<?> decodedCoder = CoderTranslation.fromProto(coderProto, encodedComponents);
       assertThat(decodedCoder, Matchers.<Coder<?>>equalTo(coder));
 
       if (KNOWN_CODERS.contains(coder)) {
         for (RunnerApi.Coder encodedCoder : encodedComponents.getCodersMap().values()) {
           assertThat(
               encodedCoder.getSpec().getSpec().getUrn(),
-              not(equalTo(Coders.JAVA_SERIALIZED_CODER_URN)));
+              not(equalTo(CoderTranslation.JAVA_SERIALIZED_CODER_URN)));
         }
       }
     }

From c8b2119ab9a75c7f781ce73ea9352734640a6f46 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:32:47 -0700
Subject: [PATCH 311/387] Rename WindowingStrategies to
 WindowingStrategyTranslation

---
 .../core/construction/PCollectionTranslation.java     |  2 +-
 .../runners/core/construction/ParDoTranslation.java   |  2 +-
 .../beam/runners/core/construction/SdkComponents.java |  2 +-
 .../core/construction/WindowIntoTranslation.java      |  4 ++--
 ...ategies.java => WindowingStrategyTranslation.java} |  2 +-
 ...est.java => WindowingStrategyTranslationTest.java} | 11 ++++++-----
 .../runners/dataflow/DataflowPipelineTranslator.java  |  4 ++--
 7 files changed, 14 insertions(+), 13 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{WindowingStrategies.java => WindowingStrategyTranslation.java} (99%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{WindowingStrategiesTest.java => WindowingStrategyTranslationTest.java} (91%)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
index 46f714e66ef5..303c02d44a11 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
@@ -60,7 +60,7 @@ public static Coder<?> getCoder(
   public static WindowingStrategy<?, ?> getWindowingStrategy(
       RunnerApi.PCollection pCollection, RunnerApi.Components components)
       throws InvalidProtocolBufferException {
-    return WindowingStrategies.fromProto(
+    return WindowingStrategyTranslation.fromProto(
         components.getWindowingStrategiesOrThrow(pCollection.getWindowingStrategyId()), components);
   }
 
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
index bc5bb0e414a3..28d577ff2211 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -265,7 +265,7 @@ public static PCollectionView<?> fromProto(
     RunnerApi.PCollection inputCollection =
         components.getPcollectionsOrThrow(parDoTransform.getInputsOrThrow(id));
     WindowingStrategy<?, ?> windowingStrategy =
-        WindowingStrategies.fromProto(
+        WindowingStrategyTranslation.fromProto(
             components.getWindowingStrategiesOrThrow(inputCollection.getWindowingStrategyId()),
             components);
     Coder<?> elemCoder =
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index 5c8187513402..b0f164f4bcbd 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -200,7 +200,7 @@ String registerWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy) thro
     String name = uniqify(baseName, windowingStrategyIds.values());
     windowingStrategyIds.put(windowingStrategy, name);
     RunnerApi.WindowingStrategy windowingStrategyProto =
-        WindowingStrategies.toProto(windowingStrategy, this);
+        WindowingStrategyTranslation.toProto(windowingStrategy, this);
     componentsBuilder.putWindowingStrategies(name, windowingStrategyProto);
     return name;
   }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
index 69793b56d83c..215beba43cd8 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
@@ -49,13 +49,13 @@ public FunctionSpec translate(
 
   public static WindowIntoPayload toProto(Window.Assign<?> transform, SdkComponents components) {
     return WindowIntoPayload.newBuilder()
-        .setWindowFn(WindowingStrategies.toProto(transform.getWindowFn(), components))
+        .setWindowFn(WindowingStrategyTranslation.toProto(transform.getWindowFn(), components))
         .build();
   }
 
   public static WindowFn<?, ?> getWindowFn(WindowIntoPayload payload)
       throws InvalidProtocolBufferException {
     SdkFunctionSpec spec = payload.getWindowFn();
-    return WindowingStrategies.windowFnFromProto(spec);
+    return WindowingStrategyTranslation.windowFnFromProto(spec);
   }
 }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
similarity index 99%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
index 8dceebbe6cc3..061f30961fd8 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategies.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
@@ -40,7 +40,7 @@
 import org.joda.time.Duration;
 
 /** Utilities for working with {@link WindowingStrategy WindowingStrategies}. */
-public class WindowingStrategies implements Serializable {
+public class WindowingStrategyTranslation implements Serializable {
 
   public static AccumulationMode fromProto(RunnerApi.AccumulationMode proto) {
     switch (proto) {
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
similarity index 91%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
index 7296a7789943..1e528039562c 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategiesTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java
@@ -41,7 +41,7 @@
 
 /** Unit tests for {@link WindowingStrategy}. */
 @RunWith(Parameterized.class)
-public class WindowingStrategiesTest {
+public class WindowingStrategyTranslationTest {
 
   // Each spec activates tests of all subsets of its fields
   @AutoValue
@@ -50,7 +50,7 @@ abstract static class ToProtoAndBackSpec {
   }
 
   private static ToProtoAndBackSpec toProtoAndBackSpec(WindowingStrategy windowingStrategy) {
-    return new AutoValue_WindowingStrategiesTest_ToProtoAndBackSpec(windowingStrategy);
+    return new AutoValue_WindowingStrategyTranslationTest_ToProtoAndBackSpec(windowingStrategy);
   }
 
   private static final WindowFn<?, ?> REPRESENTATIVE_WINDOW_FN =
@@ -85,7 +85,8 @@ public static Iterable<ToProtoAndBackSpec> data() {
   public void testToProtoAndBack() throws Exception {
     WindowingStrategy<?, ?> windowingStrategy = toProtoAndBackSpec.getWindowingStrategy();
     WindowingStrategy<?, ?> toProtoAndBackWindowingStrategy =
-        WindowingStrategies.fromProto(WindowingStrategies.toProto(windowingStrategy));
+        WindowingStrategyTranslation.fromProto(
+            WindowingStrategyTranslation.toProto(windowingStrategy));
 
     assertThat(
         toProtoAndBackWindowingStrategy,
@@ -97,11 +98,11 @@ public void testToProtoAndBackWithComponents() throws Exception {
     WindowingStrategy<?, ?> windowingStrategy = toProtoAndBackSpec.getWindowingStrategy();
     SdkComponents components = SdkComponents.create();
     RunnerApi.WindowingStrategy proto =
-        WindowingStrategies.toProto(windowingStrategy, components);
+        WindowingStrategyTranslation.toProto(windowingStrategy, components);
     RunnerApi.Components protoComponents = components.toComponents();
 
     assertThat(
-        WindowingStrategies.fromProto(proto, protoComponents).fixDefaults(),
+        WindowingStrategyTranslation.fromProto(proto, protoComponents).fixDefaults(),
         Matchers.<WindowingStrategy<?, ?>>equalTo(windowingStrategy.fixDefaults()));
 
     protoComponents.getCodersOrThrow(
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 6d7a0f847ca2..af93ef59bba4 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -56,7 +56,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.Nullable;
-import org.apache.beam.runners.core.construction.WindowingStrategies;
+import org.apache.beam.runners.core.construction.WindowingStrategyTranslation;
 import org.apache.beam.runners.dataflow.BatchViewOverrides.GroupByKeyAndSortValuesOnly;
 import org.apache.beam.runners.dataflow.DataflowRunner.CombineGroupedValues;
 import org.apache.beam.runners.dataflow.PrimitiveParDoSingleFactory.ParDoSingle;
@@ -124,7 +124,7 @@ public class DataflowPipelineTranslator {
 
   private static byte[] serializeWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy) {
     try {
-      return WindowingStrategies.toProto(windowingStrategy).toByteArray();
+      return WindowingStrategyTranslation.toProto(windowingStrategy).toByteArray();
     } catch (Exception e) {
       throw new RuntimeException(
           String.format("Unable to format windowing strategy %s as bytes", windowingStrategy), e);

From 4fa38e2d590cc4472c119b57790f7a724a700e43 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 15:33:33 -0700
Subject: [PATCH 312/387] Rename Triggers to TriggerTranslation

---
 .../translation/operators/ApexGroupByKeyOperator.java  |  4 ++--
 .../{Triggers.java => TriggerTranslation.java}         |  6 +++---
 .../construction/WindowingStrategyTranslation.java     |  4 ++--
 .../{TriggersTest.java => TriggerTranslationTest.java} |  9 +++++----
 .../core/GroupAlsoByWindowViaWindowSetNewDoFn.java     |  4 ++--
 .../org/apache/beam/runners/core/ReduceFnTester.java   | 10 ++++++----
 .../direct/GroupAlsoByWindowEvaluatorFactory.java      |  4 ++--
 .../stateful/SparkGroupAlsoByWindowViaWindowSet.java   |  4 ++--
 .../SparkGroupAlsoByWindowViaOutputBufferFn.java       |  4 ++--
 9 files changed, 26 insertions(+), 23 deletions(-)
 rename runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/{Triggers.java => TriggerTranslation.java} (98%)
 rename runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/{TriggersTest.java => TriggerTranslationTest.java} (94%)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
index 1d48e2032d8a..39f681fcb821 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators/ApexGroupByKeyOperator.java
@@ -41,7 +41,7 @@
 import org.apache.beam.runners.core.SystemReduceFn;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
 import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.sdk.coders.Coder;
@@ -163,7 +163,7 @@ private ReduceFnRunner<K, V, Iterable<V>, BoundedWindow> newReduceFnRunner(K key
         windowingStrategy,
         ExecutableTriggerStateMachine.create(
             TriggerStateMachines.stateMachineForTrigger(
-                Triggers.toProto(windowingStrategy.getTrigger()))),
+                TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
         stateInternalsFactory.stateInternalsForKey(key),
         timerInternals,
         new OutputWindowedValue<KV<K, Iterable<V>>>() {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
similarity index 98%
rename from runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
rename to runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
index df6c9ed1fa04..777b165a9b51 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Triggers.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java
@@ -47,9 +47,9 @@
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
-/** Utilities for working with {@link Triggers Triggers}. */
+/** Utilities for working with {@link TriggerTranslation Triggers}. */
 @Experimental(Experimental.Kind.TRIGGER)
-public class Triggers implements Serializable {
+public class TriggerTranslation implements Serializable {
 
   @VisibleForTesting static final ProtoConverter CONVERTER = new ProtoConverter();
 
@@ -332,5 +332,5 @@ private static List<Trigger> protosToTriggers(List<RunnerApi.Trigger> triggers)
   }
 
   // Do not instantiate
-  private Triggers() {}
+  private TriggerTranslation() {}
 }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
index 061f30961fd8..e92565f1a783 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java
@@ -210,7 +210,7 @@ public static RunnerApi.WindowingStrategy toProto(
             .setAccumulationMode(toProto(windowingStrategy.getMode()))
             .setClosingBehavior(toProto(windowingStrategy.getClosingBehavior()))
             .setAllowedLateness(windowingStrategy.getAllowedLateness().getMillis())
-            .setTrigger(Triggers.toProto(windowingStrategy.getTrigger()))
+            .setTrigger(TriggerTranslation.toProto(windowingStrategy.getTrigger()))
             .setWindowFn(windowFnSpec)
             .setWindowCoderId(
                 components.registerCoder(windowingStrategy.getWindowFn().windowCoder()));
@@ -247,7 +247,7 @@ public static RunnerApi.WindowingStrategy toProto(
     WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec);
     TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
     AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
-    Trigger trigger = Triggers.fromProto(proto.getTrigger());
+    Trigger trigger = TriggerTranslation.fromProto(proto.getTrigger());
     ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
     Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
 
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java
similarity index 94%
rename from runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggersTest.java
rename to runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java
index cf9d40c8c9cf..55ea87bca9e3 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java
@@ -41,9 +41,9 @@
 import org.junit.runners.Parameterized.Parameter;
 import org.junit.runners.Parameterized.Parameters;
 
-/** Tests for utilities in {@link Triggers}. */
+/** Tests for utilities in {@link TriggerTranslation}. */
 @RunWith(Parameterized.class)
-public class TriggersTest {
+public class TriggerTranslationTest {
 
   @AutoValue
   abstract static class ToProtoAndBackSpec {
@@ -51,7 +51,7 @@ abstract static class ToProtoAndBackSpec {
   }
 
   private static ToProtoAndBackSpec toProtoAndBackSpec(Trigger trigger) {
-    return new AutoValue_TriggersTest_ToProtoAndBackSpec(trigger);
+    return new AutoValue_TriggerTranslationTest_ToProtoAndBackSpec(trigger);
   }
 
   @Parameters(name = "{index}: {0}")
@@ -104,7 +104,8 @@ public static Iterable<ToProtoAndBackSpec> data() {
   @Test
   public void testToProtoAndBack() throws Exception {
     Trigger trigger = toProtoAndBackSpec.getTrigger();
-    Trigger toProtoAndBackTrigger = Triggers.fromProto(Triggers.toProto(trigger));
+    Trigger toProtoAndBackTrigger =
+        TriggerTranslation.fromProto(TriggerTranslation.toProto(trigger));
 
     assertThat(toProtoAndBackTrigger, equalTo(trigger));
   }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
index 744d1625667d..0a520bdb513d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupAlsoByWindowViaWindowSetNewDoFn.java
@@ -18,7 +18,7 @@
 package org.apache.beam.runners.core;
 
 import java.util.Collection;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
 import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -122,7 +122,7 @@ public void processElement(ProcessContext c) throws Exception {
             windowingStrategy,
             ExecutableTriggerStateMachine.create(
                 TriggerStateMachines.stateMachineForTrigger(
-                    Triggers.toProto(windowingStrategy.getTrigger()))),
+                    TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
             stateInternals,
             timerInternals,
             outputWindowedValue(),
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
index 7de8f3b28dfd..7f83eae78724 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
@@ -38,7 +38,7 @@
 import java.util.Set;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.core.TimerInternals.TimerData;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
 import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
 import org.apache.beam.runners.core.triggers.TriggerStateMachine;
 import org.apache.beam.runners.core.triggers.TriggerStateMachineRunner;
@@ -116,7 +116,7 @@ public class ReduceFnTester<InputT, OutputT, W extends BoundedWindow> {
     return new ReduceFnTester<Integer, Iterable<Integer>, W>(
         windowingStrategy,
         TriggerStateMachines.stateMachineForTrigger(
-            Triggers.toProto(windowingStrategy.getTrigger())),
+            TriggerTranslation.toProto(windowingStrategy.getTrigger())),
         SystemReduceFn.<String, Integer, W>buffering(VarIntCoder.of()),
         IterableCoder.of(VarIntCoder.of()),
         PipelineOptionsFactory.create(),
@@ -179,7 +179,8 @@ ReduceFnTester<Integer, OutputT, W> combining(
 
     return combining(
         strategy,
-        TriggerStateMachines.stateMachineForTrigger(Triggers.toProto(strategy.getTrigger())),
+        TriggerStateMachines.stateMachineForTrigger(
+            TriggerTranslation.toProto(strategy.getTrigger())),
         combineFn,
         outputCoder);
   }
@@ -227,7 +228,8 @@ ReduceFnTester<Integer, OutputT, W> combining(
 
     return combining(
         strategy,
-        TriggerStateMachines.stateMachineForTrigger(Triggers.toProto(strategy.getTrigger())),
+        TriggerStateMachines.stateMachineForTrigger(
+            TriggerTranslation.toProto(strategy.getTrigger())),
         combineFn,
         outputCoder,
         options,
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
index 1a588eedf895..a944e758cc6d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -32,7 +32,7 @@
 import org.apache.beam.runners.core.SystemReduceFn;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.UnsupportedSideInputReader;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
 import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
@@ -162,7 +162,7 @@ public void processElement(WindowedValue<KeyedWorkItem<K, V>> element) throws Ex
           (CopyOnAccessInMemoryStateInternals) stepContext.stateInternals();
       DirectTimerInternals timerInternals = stepContext.timerInternals();
       RunnerApi.Trigger runnerApiTrigger =
-          Triggers.toProto(windowingStrategy.getTrigger());
+          TriggerTranslation.toProto(windowingStrategy.getTrigger());
       ReduceFnRunner<K, V, Iterable<V>, BoundedWindow> reduceFnRunner =
           new ReduceFnRunner<>(
               key,
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 815b6ba5dc02..be4f3f65a3b7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -31,7 +31,7 @@
 import org.apache.beam.runners.core.SystemReduceFn;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.UnsupportedSideInputReader;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
 import org.apache.beam.runners.core.metrics.CounterCell;
 import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
 import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
@@ -260,7 +260,7 @@ public JavaPairRDD<ByteArray, byte[]> call(
                               windowingStrategy,
                               ExecutableTriggerStateMachine.create(
                                   TriggerStateMachines.stateMachineForTrigger(
-                                      Triggers.toProto(windowingStrategy.getTrigger()))),
+                                      TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
                               stateInternals,
                               timerInternals,
                               outputHolder,
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
index be023356ba54..d2a34244e6e5 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
@@ -30,7 +30,7 @@
 import org.apache.beam.runners.core.SystemReduceFn;
 import org.apache.beam.runners.core.TimerInternals;
 import org.apache.beam.runners.core.UnsupportedSideInputReader;
-import org.apache.beam.runners.core.construction.Triggers;
+import org.apache.beam.runners.core.construction.TriggerTranslation;
 import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
 import org.apache.beam.runners.core.triggers.TriggerStateMachines;
 import org.apache.beam.runners.spark.aggregators.NamedAggregators;
@@ -92,7 +92,7 @@ public Iterable<WindowedValue<KV<K, Iterable<InputT>>>> call(
             windowingStrategy,
             ExecutableTriggerStateMachine.create(
                 TriggerStateMachines.stateMachineForTrigger(
-                    Triggers.toProto(windowingStrategy.getTrigger()))),
+                    TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
             stateInternals,
             timerInternals,
             outputter,

From efc8f425317c9c54912c78af8fbbe55586aafc1b Mon Sep 17 00:00:00 2001
From: Ahmet Altay <altay@google.com>
Date: Mon, 22 May 2017 16:40:09 -0700
Subject: [PATCH 313/387] upgrading python sdk dependencies

---
 sdks/python/setup.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 9bf3cf4d37e9..befc0249c0bc 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -94,7 +94,7 @@ def get_version():
     'httplib2>=0.8,<0.10',
     'mock>=1.0.1,<3.0.0',
     'oauth2client>=2.0.1,<4.0.0',
-    'protobuf==3.2.0',
+    'protobuf>=3.2.0,<=3.3.0',
     'pyyaml>=3.12,<4.0.0',
     ]
 
@@ -103,11 +103,11 @@ def get_version():
     ]
 
 GCP_REQUIREMENTS = [
-  'google-apitools==0.5.10',
-  'proto-google-cloud-datastore-v1==0.90.0',
+  'google-apitools>=0.5.10,<=0.5.11',
+  'proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4',
   'googledatastore==7.0.1',
   # GCP packages required by tests
-  'google-cloud-bigquery>=0.23.0,<0.24.0',
+  'google-cloud-bigquery>=0.23.0,<0.25.0',
 ]
 
 

From 7fb2938e66e48160e9df878ccf9d0c2a67790151 Mon Sep 17 00:00:00 2001
From: Borisa Zivkovic <borisa.zivkovic@huawei.com>
Date: Tue, 16 May 2017 14:16:07 +0100
Subject: [PATCH 314/387] Do not ever shrink allowed lateness

This does not seem to have any use except to cause data loss.
---
 .../beam/sdk/transforms/windowing/Window.java |  6 +++-
 .../sdk/transforms/windowing/WindowTest.java  | 36 +++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index dc4863b06b0c..105ebfbe24af 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -19,6 +19,7 @@
 
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Ordering;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -141,6 +142,7 @@
  */
 @AutoValue
 public abstract class Window<T> extends PTransform<PCollection<T>, PCollection<T>>  {
+
   /**
    * Specifies the conditions under which a final pane will be created when a window is permanently
    * closed.
@@ -313,7 +315,8 @@ public Window<T> withAllowedLateness(Duration allowedLateness, ClosingBehavior b
       result = result.withMode(getAccumulationMode());
     }
     if (getAllowedLateness() != null) {
-      result = result.withAllowedLateness(getAllowedLateness());
+      result = result.withAllowedLateness(Ordering.natural().max(getAllowedLateness(),
+          inputStrategy.getAllowedLateness()));
     }
     if (getClosingBehavior() != null) {
       result = result.withClosingBehavior(getClosingBehavior());
@@ -366,6 +369,7 @@ public PCollection<T> expand(PCollection<T> input) {
 
     WindowingStrategy<?, ?> outputStrategy =
         getOutputStrategyInternal(input.getWindowingStrategy());
+
     if (getWindowFn() == null) {
       // A new PCollection must be created in case input is reused in a different location as the
       // two PCollections will, in general, have a different windowing strategy.
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index f536a9a9b8a1..65af7a1b377c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -165,6 +165,42 @@ public void testWindowIntoPropagatesLateness() {
     assertEquals(fixed25, strategy.getWindowFn());
   }
 
+  @Test
+  public void testWindowIntoAssignesLongerAllowedLateness() {
+
+    FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10));
+    FixedWindows fixed25 = FixedWindows.of(Duration.standardMinutes(25));
+
+    PCollection<String> notChanged = pipeline
+        .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of()))
+        .apply("WindowInto25", Window.<String>into(fixed25)
+            .withAllowedLateness(Duration.standardDays(1))
+            .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(5)))
+            .accumulatingFiredPanes())
+        .apply("WindowInto10", Window.<String>into(fixed10)
+            .withAllowedLateness(Duration.standardDays(2)));
+
+    assertEquals(Duration.standardDays(2), notChanged.getWindowingStrategy()
+        .getAllowedLateness());
+
+    PCollection<String> data = pipeline
+        .apply("createChanged", Create.of("hello", "world").withCoder(StringUtf8Coder.of()));
+
+    PCollection<String> longWindow = data.apply("WindowInto25c", Window.<String>into(fixed25)
+            .withAllowedLateness(Duration.standardDays(1))
+            .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(5)))
+            .accumulatingFiredPanes());
+
+    assertEquals(Duration.standardDays(1), longWindow.getWindowingStrategy()
+        .getAllowedLateness());
+
+    PCollection<String> autoCorrectedWindow = longWindow.apply("WindowInto10c",
+        Window.<String>into(fixed10).withAllowedLateness(Duration.standardHours(1)));
+
+    assertEquals(Duration.standardDays(1), autoCorrectedWindow.getWindowingStrategy()
+        .getAllowedLateness());
+  }
+
   /**
    * With {@link #testWindowIntoNullWindowFnNoAssign()}, demonstrates that the expansions of the
    * {@link Window} transform depends on if it actually assigns elements to windows.

From 6a792f3251904f191ff27e58bb9bb78b9c30fdd9 Mon Sep 17 00:00:00 2001
From: Daniel Mills <millsd@google.com>
Date: Mon, 8 May 2017 16:45:44 -0700
Subject: [PATCH 315/387] Adds large key tests to GroupByKeyTest

---
 runners/direct-java/pom.xml                   |  4 +-
 runners/flink/pom.xml                         |  2 +
 runners/google-cloud-dataflow-java/pom.xml    |  1 +
 .../apache/beam/sdk/testing/LargeKeys.java    | 48 ++++++++++++
 .../beam/sdk/transforms/GroupByKeyTest.java   | 74 +++++++++++++++++++
 5 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java

diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 857dcf42fd7b..bec21139d989 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -144,11 +144,13 @@
             </goals>
             <configuration>
               <groups>org.apache.beam.sdk.testing.NeedsRunner</groups>
+              <!-- 100MB keys work on the direct runner, but make the test too slow. --> 
+              <excludedGroups>org.apache.beam.sdk.testing.LargeKeys$Above100MB</excludedGroups>
               <parallel>none</parallel>
               <failIfNoTests>true</failIfNoTests>
               <dependenciesToScan>
                 <dependency>org.apache.beam:beam-sdks-java-core</dependency>
-                <dependency>org.apache.beam:beam-runners-core-java</dependency>
+                <dependency>org.apache.beam:beam-runners-java-core</dependency>
               </dependenciesToScan>
               <systemPropertyVariables>
                 <beamTestPipelineOptions>
diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index ff73ec1eb9d0..fb0a67c58ebc 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -57,6 +57,7 @@
                   <groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
                   <excludedGroups>
                     org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
+                    org.apache.beam.sdk.testing.LargeKeys$Above100MB,
                     org.apache.beam.sdk.testing.UsesSplittableParDo,
                     org.apache.beam.sdk.testing.UsesCommittedMetrics,
                     org.apache.beam.sdk.testing.UsesTestStream
@@ -89,6 +90,7 @@
                   <groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
                   <excludedGroups>
                     org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
+                    org.apache.beam.sdk.testing.LargeKeys$Above100MB,
                     org.apache.beam.sdk.testing.UsesSetState,
                     org.apache.beam.sdk.testing.UsesMapState,
                     org.apache.beam.sdk.testing.UsesCommittedMetrics,
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 895a8e6a513b..16e12664918c 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -217,6 +217,7 @@
             <id>validates-runner-tests</id>
             <configuration>
               <excludedGroups>
+                org.apache.beam.sdk.testing.LargeKeys$Above10MB,
                 org.apache.beam.sdk.testing.UsesDistributionMetrics,
                 org.apache.beam.sdk.testing.UsesGaugeMetrics,
                 org.apache.beam.sdk.testing.UsesSetState,
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java
new file mode 100644
index 000000000000..384b2980f47c
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/LargeKeys.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.testing;
+
+/**
+ * Category tags for tests which validate that a Beam runner can handle keys up to a given size.
+ */
+public interface LargeKeys {
+  /**
+   * Tests if a runner supports 10KB keys.
+   */
+  public interface Above10KB {}
+
+  /**
+   * Tests if a runner supports 100KB keys.
+   */
+  public interface Above100KB extends Above10KB {}
+
+  /**
+   * Tests if a runner supports 1MB keys.
+   */
+  public interface Above1MB extends Above100KB {}
+
+  /**
+   * Tests if a runner supports 10MB keys.
+   */
+  public interface Above10MB extends Above1MB {}
+
+  /**
+   * Tests if a runner supports 100MB keys.
+   */
+  public interface Above100MB extends Above10MB {}
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
index 0cd885ca2557..171171f33cd4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
@@ -45,6 +45,7 @@
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.MapCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.testing.LargeKeys;
 import org.apache.beam.sdk.testing.NeedsRunner;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -427,6 +428,79 @@ public void testGroupByKeyWithBadEqualsHashCode() throws Exception {
     p.run();
   }
 
+  private static String bigString(char c, int size) {
+    char[] buf = new char[size];
+    for (int i = 0; i < size; i++) {
+      buf[i] = c;
+    }
+    return new String(buf);
+  }
+
+  private static void runLargeKeysTest(TestPipeline p, final int keySize) throws Exception {
+    PCollection<KV<String, Integer>> result = p
+        .apply(Create.of("a", "a", "b"))
+        .apply("Expand", ParDo.of(new DoFn<String, KV<String, String>>() {
+              @ProcessElement
+              public void process(ProcessContext c) {
+                c.output(KV.of(bigString(c.element().charAt(0), keySize), c.element()));
+              }
+          }))
+        .apply(GroupByKey.<String, String>create())
+        .apply("Count", ParDo.of(new DoFn<KV<String, Iterable<String>>, KV<String, Integer>>() {
+              @ProcessElement
+              public void process(ProcessContext c) {
+                int size = 0;
+                for (String value : c.element().getValue()) {
+                  size++;
+                }
+                c.output(KV.of(c.element().getKey(), size));
+              }
+          }));
+
+    PAssert.that(result).satisfies(
+        new SerializableFunction<Iterable<KV<String, Integer>>, Void>() {
+          @Override
+          public Void apply(Iterable<KV<String, Integer>> values) {
+            assertThat(values,
+                containsInAnyOrder(
+                    KV.of(bigString('a', keySize), 2), KV.of(bigString('b', keySize), 1)));
+            return null;
+          }
+        });
+
+    p.run();
+  }
+
+  @Test
+  @Category({ValidatesRunner.class, LargeKeys.Above10KB.class})
+  public void testLargeKeys10KB() throws Exception {
+    runLargeKeysTest(p, 10 << 10);
+  }
+
+  @Test
+  @Category({ValidatesRunner.class, LargeKeys.Above100KB.class})
+  public void testLargeKeys100KB() throws Exception {
+    runLargeKeysTest(p, 100 << 10);
+  }
+
+  @Test
+  @Category({ValidatesRunner.class, LargeKeys.Above1MB.class})
+  public void testLargeKeys1MB() throws Exception {
+    runLargeKeysTest(p, 1 << 20);
+  }
+
+  @Test
+  @Category({ValidatesRunner.class, LargeKeys.Above10MB.class})
+  public void testLargeKeys10MB() throws Exception {
+    runLargeKeysTest(p, 10 << 20);
+  }
+
+  @Test
+  @Category({ValidatesRunner.class, LargeKeys.Above100MB.class})
+  public void testLargeKeys100MB() throws Exception {
+    runLargeKeysTest(p, 100 << 20);
+  }
+
   /**
    * This is a bogus key class that returns random hash values from {@link #hashCode()} and always
    * returns {@code false} for {@link #equals(Object)}. The results of the test are correct if

From 5b899a8518cc8910f0a855303c14088d72b332e5 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 18 May 2017 10:23:16 -0700
Subject: [PATCH 316/387] Add CombineTranslation

This translates Combines to CombinePayloads and back
---
 .../core/construction/CombineTranslation.java | 125 +++++++++++++++++
 .../construction/CombineTranslationTest.java  | 130 ++++++++++++++++++
 .../org/apache/beam/sdk/transforms/Count.java |  10 ++
 .../org/apache/beam/sdk/transforms/Sum.java   |  30 ++++
 4 files changed, 295 insertions(+)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
 create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
new file mode 100644
index 000000000000..e0b6d5c9abf8
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Iterables;
+import com.google.protobuf.Any;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.BytesValue;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.CombinePayload;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.AppliedCombineFn;
+import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Methods for translating between {@link Combine.PerKey} {@link PTransform PTransforms} and {@link
+ * RunnerApi.CombinePayload} protos.
+ */
+public class CombineTranslation {
+  private static final String JAVA_SERIALIZED_COMBINE_FN_URN = "urn:beam:java:combinefn:v1";
+
+  public static CombinePayload toProto(
+      AppliedPTransform<?, ?, Combine.PerKey<?, ?, ?>> combine, SdkComponents sdkComponents)
+      throws IOException {
+    GlobalCombineFn<?, ?, ?> combineFn = combine.getTransform().getFn();
+    try {
+      Coder<?> accumulatorCoder = extractAccumulatorCoder(combineFn, (AppliedPTransform) combine);
+      Map<String, SideInput> sideInputs = new HashMap<>();
+      return RunnerApi.CombinePayload.newBuilder()
+          .setAccumulatorCoderId(sdkComponents.registerCoder(accumulatorCoder))
+          .putAllSideInputs(sideInputs)
+          .setCombineFn(toProto(combineFn))
+          .build();
+    } catch (CannotProvideCoderException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+
+  private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
+      GlobalCombineFn<InputT, AccumT, ?> combineFn,
+      AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform)
+      throws CannotProvideCoderException {
+    KvCoder<K, InputT> inputCoder =
+        (KvCoder<K, InputT>)
+            ((PCollection<KV<K, InputT>>) Iterables.getOnlyElement(transform.getInputs().values()))
+                .getCoder();
+    return AppliedCombineFn.withInputCoder(
+            combineFn,
+            transform.getPipeline().getCoderRegistry(),
+            inputCoder,
+            transform.getTransform().getSideInputs(),
+            ((PCollection<?>) Iterables.getOnlyElement(transform.getOutputs().values()))
+                .getWindowingStrategy())
+        .getAccumulatorCoder();
+  }
+
+  private static SdkFunctionSpec toProto(GlobalCombineFn<?, ?, ?> combineFn) {
+    return SdkFunctionSpec.newBuilder()
+        // TODO: Set Java SDK Environment URN
+        .setSpec(
+            FunctionSpec.newBuilder()
+                .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN)
+                .setParameter(
+                    Any.pack(
+                        BytesValue.newBuilder()
+                            .setValue(
+                                ByteString.copyFrom(
+                                    SerializableUtils.serializeToByteArray(combineFn)))
+                            .build())))
+        .build();
+  }
+
+  public static Coder<?> getAccumulatorCoder(
+      CombinePayload payload, RunnerApi.Components components) throws IOException {
+    String id = payload.getAccumulatorCoderId();
+    return Coders.fromProto(components.getCodersOrThrow(id), components);
+  }
+
+  public static GlobalCombineFn<?, ?, ?> getCombineFn(CombinePayload payload)
+      throws IOException {
+    checkArgument(payload.getCombineFn().getSpec().getUrn().equals(JAVA_SERIALIZED_COMBINE_FN_URN));
+    return (GlobalCombineFn<?, ?, ?>)
+        SerializableUtils.deserializeFromByteArray(
+            payload
+                .getCombineFn()
+                .getSpec()
+                .getParameter()
+                .unpack(BytesValue.class)
+                .getValue()
+                .toByteArray(),
+            "CombineFn");
+  }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
new file mode 100644
index 000000000000..6251545b9b7c
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkState;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.CombinePayload;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Combine.BinaryCombineIntegerFn;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests for {@link CombineTranslation}.
+ */
+@RunWith(Parameterized.class)
+public class CombineTranslationTest {
+  @Parameters(name = "{index}: {0}")
+  public static Iterable<Combine.CombineFn<Integer, ?, ?>> params() {
+    BinaryCombineIntegerFn sum = Sum.ofIntegers();
+    CombineFn<Integer, ?, Long> count = Count.combineFn();
+    TestCombineFn test = new TestCombineFn();
+    return ImmutableList.<CombineFn<Integer, ?, ?>>builder().add(sum).add(count).add(test).build();
+  }
+
+  @Rule public TestPipeline pipeline = TestPipeline.create();
+  @Parameter(0)
+  public Combine.CombineFn<Integer, ?, ?> combineFn;
+
+  @Test
+  public void testToFromProto() throws Exception {
+    PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3));
+    input.apply(Combine.globally(combineFn));
+    final AtomicReference<AppliedPTransform<?, ?, Combine.PerKey<?, ?, ?>>> combine =
+        new AtomicReference<>();
+    pipeline.traverseTopologically(
+        new PipelineVisitor.Defaults() {
+          @Override
+          public void leaveCompositeTransform(Node node) {
+            if (node.getTransform() instanceof Combine.PerKey) {
+              checkState(combine.get() == null);
+              combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline()));
+            }
+          }
+        });
+    checkState(combine.get() != null);
+
+    SdkComponents sdkComponents = SdkComponents.create();
+    CombinePayload combineProto = CombineTranslation.toProto(combine.get(), sdkComponents);
+    RunnerApi.Components componentsProto = sdkComponents.toComponents();
+
+    assertEquals(
+        combineFn.getAccumulatorCoder(pipeline.getCoderRegistry(), input.getCoder()),
+        CombineTranslation.getAccumulatorCoder(combineProto, componentsProto));
+    assertEquals(combineFn, CombineTranslation.getCombineFn(combineProto));
+  }
+
+  private static class TestCombineFn extends Combine.CombineFn<Integer, Void, Void> {
+    @Override
+    public Void createAccumulator() {
+      return null;
+    }
+
+    @Override
+    public Coder<Void> getAccumulatorCoder(CoderRegistry registry, Coder<Integer> inputCoder) {
+      return (Coder) VoidCoder.of();
+    }
+
+    @Override
+    public Void extractOutput(Void accumulator) {
+      return accumulator;
+    }
+
+    @Override
+    public Void mergeAccumulators(Iterable<Void> accumulators) {
+      return null;
+    }
+
+    @Override
+    public Void addInput(Void accumulator, Integer input) {
+      return accumulator;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && other.getClass().equals(TestCombineFn.class);
+    }
+
+    @Override
+    public int hashCode() {
+      return TestCombineFn.class.hashCode();
+    }
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
index b405dd1aae30..ee24b3f1ef4b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Count.java
@@ -195,5 +195,15 @@ protected long getEncodedElementByteSize(long[] value) {
         }
       };
     }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && getClass().equals(other.getClass());
+    }
+
+    @Override
+    public int hashCode() {
+      return getClass().hashCode();
+    }
   }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java
index ccade4db703a..6b6541606d20 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sum.java
@@ -151,6 +151,16 @@ public int apply(int a, int b) {
     public int identity() {
       return 0;
     }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && other.getClass().equals(this.getClass());
+    }
+
+    @Override
+    public int hashCode() {
+      return getClass().hashCode();
+    }
   }
 
   private static class SumLongFn extends Combine.BinaryCombineLongFn {
@@ -164,6 +174,16 @@ public long apply(long a, long b) {
     public long identity() {
       return 0;
     }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && other.getClass().equals(this.getClass());
+    }
+
+    @Override
+    public int hashCode() {
+      return getClass().hashCode();
+    }
   }
 
   private static class SumDoubleFn extends Combine.BinaryCombineDoubleFn {
@@ -177,5 +197,15 @@ public double apply(double a, double b) {
     public double identity() {
       return 0;
     }
+
+    @Override
+    public boolean equals(Object other) {
+      return other != null && other.getClass().equals(this.getClass());
+    }
+
+    @Override
+    public int hashCode() {
+      return getClass().hashCode();
+    }
   }
 }

From 9cd6c3bdefd50473f9fc6fac359213dcd6b4e4d4 Mon Sep 17 00:00:00 2001
From: MOLIG004 <guy.molinari@disney.com>
Date: Thu, 4 May 2017 09:21:23 -0700
Subject: [PATCH 317/387] Initial implementation of SpannerIO.Write

This closes #2166.
---
 .../examples/spanner/SpannerCSVLoader.java    | 143 +++++++++
 pom.xml                                       |   8 +
 sdks/java/io/google-cloud-platform/pom.xml    |  33 ++-
 .../beam/sdk/io/gcp/spanner/SpannerIO.java    | 272 ++++++++++++++++++
 .../beam/sdk/io/gcp/spanner/package-info.java |  23 ++
 .../beam/sdk/io/gcp/GcpApiSurfaceTest.java    |   2 +
 6 files changed, 471 insertions(+), 10 deletions(-)
 create mode 100644 examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java

diff --git a/examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java b/examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java
new file mode 100644
index 000000000000..eee581da2148
--- /dev/null
+++ b/examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.examples.spanner;
+
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Operation;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerException;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
+import java.util.Collections;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.io.gcp.spanner.SpannerIO;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.Validation;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+
+
+
+/**
+ * Generalized bulk loader for importing CSV files into Spanner.
+ *
+ */
+public class SpannerCSVLoader {
+
+    /**
+     * Command options specification.
+     */
+    private interface Options extends PipelineOptions {
+    @Description("Create a sample database")
+    @Default.Boolean(false)
+    boolean isCreateDatabase();
+    void setCreateDatabase(boolean createDatabase);
+
+    @Description("File to read from ")
+    @Validation.Required
+    String getInput();
+    void setInput(String value);
+
+    @Description("Instance ID to write to in Spanner")
+    @Validation.Required
+    String getInstanceId();
+    void setInstanceId(String value);
+
+    @Description("Database ID to write to in Spanner")
+    @Validation.Required
+    String getDatabaseId();
+    void setDatabaseId(String value);
+
+    @Description("Table name")
+    @Validation.Required
+    String getTable();
+    void setTable(String value);
+  }
+
+
+  /**
+   * Constructs and executes the processing pipeline based upon command options.
+   */
+  public static void main(String[] args) throws Exception {
+      Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
+
+      Pipeline p = Pipeline.create(options);
+      PCollection<String> lines = p.apply(TextIO.Read.from(options.getInput()));
+      PCollection<Mutation> mutations = lines
+              .apply(ParDo.of(new NaiveParseCsvFn(options.getTable())));
+      mutations
+              .apply(SpannerIO.writeTo(options.getInstanceId(), options.getDatabaseId()));
+      p.run().waitUntilFinish();
+  }
+
+  public static void createDatabase(Options options) {
+      Spanner client = SpannerOptions.getDefaultInstance().getService();
+
+      DatabaseAdminClient databaseAdminClient = client.getDatabaseAdminClient();
+      try {
+          databaseAdminClient.dropDatabase(options.getInstanceId(), options
+                  .getDatabaseId());
+      } catch (SpannerException e) {
+          // Does not exist, ignore.
+      }
+      Operation<Database, CreateDatabaseMetadata> op = databaseAdminClient.createDatabase(
+               options.getInstanceId(), options
+              .getDatabaseId(), Collections.singleton("CREATE TABLE " + options.getTable() + " ("
+              + "  Key           INT64,"
+              + "  Name          STRING,"
+              + "  Email         STRING,"
+              + "  Age           INT,"
+              + ") PRIMARY KEY (Key)"));
+      op.waitFor();
+  }
+
+
+  /**
+   * A DoFn that creates a Spanner Mutation for each CSV line.
+   */
+  static class NaiveParseCsvFn extends DoFn<String, Mutation> {
+      private final String table;
+
+      NaiveParseCsvFn(String table) {
+          this.table = table;
+      }
+
+      @ProcessElement
+      public void processElement(ProcessContext c) {
+          String line = c.element();
+          String[] elements = line.split(",");
+          if (elements.length != 4) {
+              return;
+          }
+          Mutation mutation = Mutation.newInsertOrUpdateBuilder(table)
+                  .set("Key").to(Long.valueOf(elements[0]))
+                  .set("Name").to(elements[1])
+                  .set("Email").to(elements[2])
+                  .set("Age").to(Integer.valueOf(elements[3]))
+                  .build();
+          c.output(mutation);
+      }
+  }
+}
diff --git a/pom.xml b/pom.xml
index cc2e4837a248..3a6289dfb7f2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -104,6 +104,7 @@
     <apache.commons.lang.version>3.5</apache.commons.lang.version>
     <apache.commons.compress.version>1.9</apache.commons.compress.version>
     <apex.kryo.version>2.24.0</apex.kryo.version>
+    <api-common.version>1.0.0-rc2</api-common.version>
     <avro.version>1.8.1</avro.version>
     <bigquery.version>v2-rev295-1.22.0</bigquery.version>
     <bigtable.version>0.9.6.2</bigtable.version>
@@ -139,6 +140,7 @@
     <stax2.version>3.1.4</stax2.version>
     <storage.version>v1-rev71-1.22.0</storage.version>
     <woodstox.version>4.4.1</woodstox.version>
+    <spanner.version>0.16.0-beta</spanner.version>
     <spring.version>4.3.5.RELEASE</spring.version>
     <groovy-maven-plugin.version>2.0</groovy-maven-plugin.version>
     <snappy-java.version>1.1.4-M3</snappy-java.version>
@@ -865,6 +867,12 @@
         <version>${google-cloud-bigdataoss.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-spanner</artifactId>
+        <version>${spanner.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.google.cloud.bigdataoss</groupId>
         <artifactId>util</artifactId>
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index ea2d8f0cb290..21818959843f 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -81,11 +81,28 @@
       <artifactId>jackson-databind</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-core</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>grpc-google-common-protos</artifactId>
+      <version>${grpc-google-common-protos.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-bigquery</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.api</groupId>
+      <artifactId>api-common</artifactId>
+      <version>${api-common.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-pubsub</artifactId>
@@ -116,11 +133,6 @@
       <artifactId>grpc-auth</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>io.grpc</groupId>
-      <artifactId>grpc-core</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>io.grpc</groupId>
       <artifactId>grpc-netty</artifactId>
@@ -149,6 +161,12 @@
       <artifactId>joda-time</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-spanner</artifactId>
+      <version>${spanner.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>com.google.cloud.bigtable</groupId>
       <artifactId>bigtable-protos</artifactId>
@@ -184,11 +202,6 @@
       <artifactId>google-auth-library-oauth2-http</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.api.grpc</groupId>
-      <artifactId>grpc-google-common-protos</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
new file mode 100644
index 000000000000..172ed8f02687
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.api.client.util.BackOff;
+import com.google.api.client.util.BackOffUtils;
+import com.google.api.client.util.Sleeper;
+
+import com.google.cloud.spanner.AbortedException;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
+import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+
+/**
+ *  <a href="https://cloud.google.com/spanner/">Google Cloud Spanner</a> connectors.
+ *
+ * <h3>Reading from Cloud Spanner</h3>
+ * <strong>Status: Not implemented.</strong>
+ *
+ * <h3>Writing to Cloud Spanner</h3>
+ * <strong>Status: Experimental.</strong>
+ *
+ * <p>{@link SpannerIO#writeTo} batches together and concurrently writes a set of {@link Mutation}s.
+ * To configure Cloud Spanner sink, you must apply {@link SpannerIO#writeTo} transform to
+ * {@link PCollection} and specify instance and database identifiers.
+ * For example, following code sketches out a pipeline that imports data from the CSV file to Cloud
+ * Spanner.
+ *
+ * <pre>{@code
+ *
+ * Pipeline p = ...;
+ * // Read the CSV file.
+ * PCollection<String> lines = p.apply("Read CSV file", TextIO.Read.from(options.getInput()));
+ * // Parse the line and convert to mutation.
+ * PCollection<Mutation> mutations = lines.apply("Parse CSV", parseFromCsv());
+ * // Write mutations.
+ * mutations.apply("Write", SpannerIO.writeTo(options.getInstanceId(), options.getDatabaseId()));
+ * p.run();
+ *
+ * }</pre>
+
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class SpannerIO {
+
+  private SpannerIO() {
+  }
+
+  @VisibleForTesting
+  static final int SPANNER_MUTATIONS_PER_COMMIT_LIMIT = 20000;
+
+    /**
+     * Creates an instance of {@link Writer}. Use {@link Writer#withBatchSize} to limit the batch
+     * size.
+     */
+  public static Writer writeTo(String instanceId, String databaseId) {
+    return new Writer(instanceId, databaseId, SPANNER_MUTATIONS_PER_COMMIT_LIMIT);
+  }
+
+  /**
+   * A {@link PTransform} that writes {@link Mutation} objects to Cloud Spanner.
+   *
+   * @see SpannerIO
+   */
+  public static class Writer extends PTransform<PCollection<Mutation>, PDone> {
+
+    private final String instanceId;
+    private final String databaseId;
+    private int batchSize;
+
+    Writer(String instanceId, String databaseId, int batchSize) {
+      this.instanceId = instanceId;
+      this.databaseId = databaseId;
+      this.batchSize = batchSize;
+    }
+
+    /**
+     * Returns a new {@link Writer} with a limit on the number of mutations per batch.
+     * Defaults to {@link SpannerIO#SPANNER_MUTATIONS_PER_COMMIT_LIMIT}.
+     */
+    public Writer withBatchSize(Integer batchSize) {
+      return new Writer(instanceId, databaseId, batchSize);
+    }
+
+    @Override
+    public PDone expand(PCollection<Mutation> input) {
+      input.apply("Write mutations to Spanner", ParDo.of(
+              new SpannerWriterFn(instanceId, databaseId, batchSize)));
+
+      return PDone.in(input.getPipeline());
+    }
+
+    @Override
+    public void validate(PCollection<Mutation> input) {
+      checkNotNull(instanceId, "instanceId");
+      checkNotNull(databaseId, "databaseId");
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(getClass())
+          .add("instanceId", instanceId)
+          .add("databaseId", databaseId)
+          .toString();
+    }
+
+    @Override
+    public void populateDisplayData(DisplayData.Builder builder) {
+      super.populateDisplayData(builder);
+      builder
+          .addIfNotNull(DisplayData.item("instanceId", instanceId)
+              .withLabel("Output Instance"))
+          .addIfNotNull(DisplayData.item("databaseId", databaseId)
+              .withLabel("Output Database"));
+    }
+
+  }
+
+
+  /**
+   * {@link DoFn} that writes {@link Mutation}s to Cloud Spanner. Mutations are written in
+   * batches, where the maximum batch size is {@link SpannerIO#SPANNER_MUTATIONS_PER_COMMIT_LIMIT}.
+   *
+   * <p>See <a href="https://cloud.google.com/spanner"/>
+   *
+   * <p>Commits are non-transactional.  If a commit fails, it will be retried (up to
+   * {@link SpannerIO#MAX_RETRIES}. times). This means that the
+   * mutation operation should be idempotent.
+   */
+  @VisibleForTesting
+  static class SpannerWriterFn extends DoFn<Mutation, Void> {
+    private static final Logger LOG = LoggerFactory.getLogger(SpannerWriterFn.class);
+    private transient Spanner spanner;
+    private final String instanceId;
+    private final String databaseId;
+    private final int batchSize;
+    private transient DatabaseClient dbClient;
+    // Current batch of mutations to be written.
+    private final List<Mutation> mutations = new ArrayList<>();
+
+    private static final int MAX_RETRIES = 5;
+    private static final FluentBackoff BUNDLE_WRITE_BACKOFF =
+        FluentBackoff.DEFAULT
+            .withMaxRetries(MAX_RETRIES).withInitialBackoff(Duration.standardSeconds(5));
+
+    @VisibleForTesting
+    SpannerWriterFn(String instanceId, String databaseId, int batchSize) {
+      this.instanceId = checkNotNull(instanceId, "instanceId");
+      this.databaseId = checkNotNull(databaseId, "databaseId");
+      this.batchSize = batchSize;
+    }
+
+    @Setup
+    public void setup() throws Exception {
+        SpannerOptions options = SpannerOptions.newBuilder().build();
+        spanner = options.getService();
+        dbClient = spanner.getDatabaseClient(
+            DatabaseId.of(options.getProjectId(), instanceId, databaseId));
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext c) throws Exception {
+      Mutation m = c.element();
+      mutations.add(m);
+      int columnCount = m.asMap().size();
+      if ((mutations.size() + 1) * columnCount >= batchSize) {
+        flushBatch();
+      }
+    }
+
+    @FinishBundle
+    public void finishBundle(Context c) throws Exception {
+      if (!mutations.isEmpty()) {
+        flushBatch();
+      }
+    }
+
+    @Teardown
+    public void teardown() throws Exception {
+      if (spanner == null) {
+          return;
+      }
+      spanner.closeAsync().get();
+    }
+
+    /**
+     * Writes a batch of mutations to Cloud Spanner.
+     *
+     * <p>If a commit fails, it will be retried up to {@link #MAX_RETRIES} times.
+     * If the retry limit is exceeded, the last exception from Cloud Spanner will be
+     * thrown.
+     *
+     * @throws AbortedException if the commit fails or IOException or InterruptedException if
+     * backing off between retries fails.
+     */
+    private void flushBatch() throws AbortedException, IOException, InterruptedException {
+      LOG.debug("Writing batch of {} mutations", mutations.size());
+      Sleeper sleeper = Sleeper.DEFAULT;
+      BackOff backoff = BUNDLE_WRITE_BACKOFF.backoff();
+
+      while (true) {
+        // Batch upsert rows.
+        try {
+          dbClient.writeAtLeastOnce(mutations);
+
+          // Break if the commit threw no exception.
+          break;
+        } catch (AbortedException exception) {
+          // Only log the code and message for potentially-transient errors. The entire exception
+          // will be propagated upon the last retry.
+          LOG.error("Error writing to Spanner ({}): {}", exception.getCode(),
+              exception.getMessage());
+          if (!BackOffUtils.next(sleeper, backoff)) {
+            LOG.error("Aborting after {} retries.", MAX_RETRIES);
+            throw exception;
+          }
+        }
+      }
+      LOG.debug("Successfully wrote {} mutations", mutations.size());
+      mutations.clear();
+    }
+
+    @Override
+    public void populateDisplayData(Builder builder) {
+      super.populateDisplayData(builder);
+      builder
+          .addIfNotNull(DisplayData.item("instanceId", instanceId)
+              .withLabel("Instance"))
+          .addIfNotNull(DisplayData.item("databaseId", databaseId)
+              .withLabel("Database"));
+    }
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java
new file mode 100644
index 000000000000..19e468cce041
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <p>Provides an API for reading from and writing to
+ * <a href="https://developers.google.com/spanner/">Google Cloud Spanner</a>.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
index 7025004cc953..8950452b5db2 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
@@ -75,6 +75,8 @@ public void testGcpApiSurface() throws Exception {
             classesInPackage("javax"),
             classesInPackage("org.apache.beam"),
             classesInPackage("org.apache.commons.logging"),
+            classesInPackage("com.google.cloud"),
+            classesInPackage("com.google.cloud.spanner"),
             // via Bigtable
             classesInPackage("org.joda.time"));
 

From e53dc915d8da30aa49fb28a632a5d890723a73dc Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 4 May 2017 10:42:07 -0700
Subject: [PATCH 318/387] Minor style, compilation, javadoc fixups

---
 .../beam/sdk/io/gcp/spanner/SpannerIO.java    | 45 +++++++++----------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
index 172ed8f02687..c9c81a51a384 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
@@ -19,10 +19,6 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.BackOffUtils;
-import com.google.api.client.util.Sleeper;
-
 import com.google.cloud.spanner.AbortedException;
 import com.google.cloud.spanner.DatabaseClient;
 import com.google.cloud.spanner.DatabaseId;
@@ -35,23 +31,25 @@
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
+import org.apache.beam.sdk.util.BackOff;
+import org.apache.beam.sdk.util.BackOffUtils;
 import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.Sleeper;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
-
-
 /**
- *  <a href="https://cloud.google.com/spanner/">Google Cloud Spanner</a> connectors.
+ * {@link PTransform Transforms} for reading from and writing to
+ * <a href="https://cloud.google.com/spanner">Google Cloud Spanner</a>.
  *
  * <h3>Reading from Cloud Spanner</h3>
  * <strong>Status: Not implemented.</strong>
@@ -61,9 +59,8 @@
  *
  * <p>{@link SpannerIO#writeTo} batches together and concurrently writes a set of {@link Mutation}s.
  * To configure Cloud Spanner sink, you must apply {@link SpannerIO#writeTo} transform to
- * {@link PCollection} and specify instance and database identifiers.
- * For example, following code sketches out a pipeline that imports data from the CSV file to Cloud
- * Spanner.
+ * {@link PCollection} and specify instance and database identifiers. For example, following code
+ * sketches out a pipeline that imports data from the CSV file to Cloud Spanner.
  *
  * <pre>{@code
  *
@@ -77,14 +74,10 @@
  * p.run();
  *
  * }</pre>
-
  */
 @Experimental(Experimental.Kind.SOURCE_SINK)
 public class SpannerIO {
 
-  private SpannerIO() {
-  }
-
   @VisibleForTesting
   static final int SPANNER_MUTATIONS_PER_COMMIT_LIMIT = 20000;
 
@@ -97,10 +90,11 @@ public static Writer writeTo(String instanceId, String databaseId) {
   }
 
   /**
-   * A {@link PTransform} that writes {@link Mutation} objects to Cloud Spanner.
+   * A {@link PTransform} that writes {@link Mutation} objects to Google Cloud Spanner.
    *
    * @see SpannerIO
    */
+  @Experimental(Experimental.Kind.SOURCE_SINK)
   public static class Writer extends PTransform<PCollection<Mutation>, PDone> {
 
     private final String instanceId;
@@ -130,7 +124,7 @@ public PDone expand(PCollection<Mutation> input) {
     }
 
     @Override
-    public void validate(PCollection<Mutation> input) {
+    public void validate(PipelineOptions options) {
       checkNotNull(instanceId, "instanceId");
       checkNotNull(databaseId, "databaseId");
     }
@@ -152,19 +146,17 @@ public void populateDisplayData(DisplayData.Builder builder) {
           .addIfNotNull(DisplayData.item("databaseId", databaseId)
               .withLabel("Output Database"));
     }
-
   }
 
-
   /**
-   * {@link DoFn} that writes {@link Mutation}s to Cloud Spanner. Mutations are written in
+   * {@link DoFn} that writes {@link Mutation}s to Google Cloud Spanner. Mutations are written in
    * batches, where the maximum batch size is {@link SpannerIO#SPANNER_MUTATIONS_PER_COMMIT_LIMIT}.
    *
-   * <p>See <a href="https://cloud.google.com/spanner"/>
-   *
    * <p>Commits are non-transactional.  If a commit fails, it will be retried (up to
-   * {@link SpannerIO#MAX_RETRIES}. times). This means that the
-   * mutation operation should be idempotent.
+   * {@link SpannerWriterFn#MAX_RETRIES} times). This means that the mutation operation should be
+   * idempotent.
+   *
+   * <p>See <a href="https://cloud.google.com/spanner">Google Cloud Spanner documentation</a>.
    */
   @VisibleForTesting
   static class SpannerWriterFn extends DoFn<Mutation, Void> {
@@ -208,7 +200,7 @@ public void processElement(ProcessContext c) throws Exception {
     }
 
     @FinishBundle
-    public void finishBundle(Context c) throws Exception {
+    public void finishBundle() throws Exception {
       if (!mutations.isEmpty()) {
         flushBatch();
       }
@@ -269,4 +261,7 @@ public void populateDisplayData(Builder builder) {
               .withLabel("Database"));
     }
   }
+
+  private SpannerIO() {} // Prevent construction.
+
 }

From 8871eadcc8413dab906656806b54f3d73b24893d Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 4 May 2017 10:48:54 -0700
Subject: [PATCH 319/387] Fix spanner dependency management

Also minor cleanup alphabetization in root pom.xml
---
 pom.xml                                    | 22 +++++++++++++++++++---
 sdks/java/io/google-cloud-platform/pom.xml |  8 +++++---
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/pom.xml b/pom.xml
index 3a6289dfb7f2..c3a6b73dfb1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,11 +115,13 @@
     <dataflow.proto.version>0.5.160222</dataflow.proto.version>
     <datastore.client.version>1.4.0</datastore.client.version>
     <datastore.proto.version>1.3.0</datastore.proto.version>
+    <google-api-common.version>1.0.0-rc2</google-api-common.version>
     <google-auto-service.version>1.0-rc2</google-auto-service.version>
     <google-auto-value.version>1.4.1</google-auto-value.version>
     <google-auth.version>0.6.1</google-auth.version>
     <google-clients.version>1.22.0</google-clients.version>
     <google-cloud-bigdataoss.version>1.4.5</google-cloud-bigdataoss.version>
+    <google-cloud-core.version>1.0.2</google-cloud-core.version>
     <google-cloud-dataflow-java-proto-library-all.version>0.5.160304</google-cloud-dataflow-java-proto-library-all.version>
     <guava.version>20.0</guava.version>
     <grpc.version>1.2.0</grpc.version>
@@ -132,18 +134,20 @@
     <mockito.version>1.9.5</mockito.version>
     <netty.version>4.1.8.Final</netty.version>
     <netty.tcnative.version>1.1.33.Fork26</netty.tcnative.version>
-    <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
     <protobuf.version>3.2.0</protobuf.version>
     <pubsub.version>v1-rev10-1.22.0</pubsub.version>
     <slf4j.version>1.7.14</slf4j.version>
+    <spanner.version>0.16.0-beta</spanner.version>
     <spark.version>1.6.2</spark.version>
+    <spring.version>4.3.5.RELEASE</spring.version>
     <stax2.version>3.1.4</stax2.version>
     <storage.version>v1-rev71-1.22.0</storage.version>
     <woodstox.version>4.4.1</woodstox.version>
-    <spanner.version>0.16.0-beta</spanner.version>
     <spring.version>4.3.5.RELEASE</spring.version>
-    <groovy-maven-plugin.version>2.0</groovy-maven-plugin.version>
     <snappy-java.version>1.1.4-M3</snappy-java.version>
+
+    <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+    <groovy-maven-plugin.version>2.0</groovy-maven-plugin.version>
     <surefire-plugin.version>2.20</surefire-plugin.version>
     <failsafe-plugin.version>2.20</failsafe-plugin.version>
     <maven-compiler-plugin.version>3.6.1</maven-compiler-plugin.version>
@@ -610,6 +614,12 @@
         <version>${grpc.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.api</groupId>
+        <artifactId>api-common</artifactId>
+        <version>${google-api-common.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.google.api-client</groupId>
         <artifactId>google-api-client</artifactId>
@@ -867,6 +877,12 @@
         <version>${google-cloud-bigdataoss.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-core</artifactId>
+        <version>${google-cloud-core.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.google.cloud</groupId>
         <artifactId>google-cloud-spanner</artifactId>
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 21818959843f..9143ccf553cf 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -89,7 +89,6 @@
     <dependency>
       <groupId>com.google.api.grpc</groupId>
       <artifactId>grpc-google-common-protos</artifactId>
-      <version>${grpc-google-common-protos.version}</version>
     </dependency>
 
     <dependency>
@@ -100,7 +99,6 @@
     <dependency>
       <groupId>com.google.api</groupId>
       <artifactId>api-common</artifactId>
-      <version>${api-common.version}</version>
     </dependency>
 
     <dependency>
@@ -161,10 +159,14 @@
       <artifactId>joda-time</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-core</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-spanner</artifactId>
-      <version>${spanner.version}</version>
     </dependency>
 
     <dependency>

From 216b972b6b09c0bb612d7f716cfec68ace9777ec Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Thu, 4 May 2017 10:48:24 -0700
Subject: [PATCH 320/387] Delete SpannerCSVLoader

This is not appropriate for examples. SpannerIO should be well-javadoced
and integration tested.
---
 .../examples/spanner/SpannerCSVLoader.java    | 143 ------------------
 1 file changed, 143 deletions(-)
 delete mode 100644 examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java

diff --git a/examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java b/examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java
deleted file mode 100644
index eee581da2148..000000000000
--- a/examples/java/src/main/java/org/apache/beam/examples/spanner/SpannerCSVLoader.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.examples.spanner;
-
-import com.google.cloud.spanner.Database;
-import com.google.cloud.spanner.DatabaseAdminClient;
-import com.google.cloud.spanner.Mutation;
-import com.google.cloud.spanner.Operation;
-import com.google.cloud.spanner.Spanner;
-import com.google.cloud.spanner.SpannerException;
-import com.google.cloud.spanner.SpannerOptions;
-import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
-import java.util.Collections;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.io.TextIO;
-import org.apache.beam.sdk.io.gcp.spanner.SpannerIO;
-import org.apache.beam.sdk.options.Default;
-import org.apache.beam.sdk.options.Description;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.options.Validation;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.values.PCollection;
-
-
-
-/**
- * Generalized bulk loader for importing CSV files into Spanner.
- *
- */
-public class SpannerCSVLoader {
-
-    /**
-     * Command options specification.
-     */
-    private interface Options extends PipelineOptions {
-    @Description("Create a sample database")
-    @Default.Boolean(false)
-    boolean isCreateDatabase();
-    void setCreateDatabase(boolean createDatabase);
-
-    @Description("File to read from ")
-    @Validation.Required
-    String getInput();
-    void setInput(String value);
-
-    @Description("Instance ID to write to in Spanner")
-    @Validation.Required
-    String getInstanceId();
-    void setInstanceId(String value);
-
-    @Description("Database ID to write to in Spanner")
-    @Validation.Required
-    String getDatabaseId();
-    void setDatabaseId(String value);
-
-    @Description("Table name")
-    @Validation.Required
-    String getTable();
-    void setTable(String value);
-  }
-
-
-  /**
-   * Constructs and executes the processing pipeline based upon command options.
-   */
-  public static void main(String[] args) throws Exception {
-      Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
-
-      Pipeline p = Pipeline.create(options);
-      PCollection<String> lines = p.apply(TextIO.Read.from(options.getInput()));
-      PCollection<Mutation> mutations = lines
-              .apply(ParDo.of(new NaiveParseCsvFn(options.getTable())));
-      mutations
-              .apply(SpannerIO.writeTo(options.getInstanceId(), options.getDatabaseId()));
-      p.run().waitUntilFinish();
-  }
-
-  public static void createDatabase(Options options) {
-      Spanner client = SpannerOptions.getDefaultInstance().getService();
-
-      DatabaseAdminClient databaseAdminClient = client.getDatabaseAdminClient();
-      try {
-          databaseAdminClient.dropDatabase(options.getInstanceId(), options
-                  .getDatabaseId());
-      } catch (SpannerException e) {
-          // Does not exist, ignore.
-      }
-      Operation<Database, CreateDatabaseMetadata> op = databaseAdminClient.createDatabase(
-               options.getInstanceId(), options
-              .getDatabaseId(), Collections.singleton("CREATE TABLE " + options.getTable() + " ("
-              + "  Key           INT64,"
-              + "  Name          STRING,"
-              + "  Email         STRING,"
-              + "  Age           INT,"
-              + ") PRIMARY KEY (Key)"));
-      op.waitFor();
-  }
-
-
-  /**
-   * A DoFn that creates a Spanner Mutation for each CSV line.
-   */
-  static class NaiveParseCsvFn extends DoFn<String, Mutation> {
-      private final String table;
-
-      NaiveParseCsvFn(String table) {
-          this.table = table;
-      }
-
-      @ProcessElement
-      public void processElement(ProcessContext c) {
-          String line = c.element();
-          String[] elements = line.split(",");
-          if (elements.length != 4) {
-              return;
-          }
-          Mutation mutation = Mutation.newInsertOrUpdateBuilder(table)
-                  .set("Key").to(Long.valueOf(elements[0]))
-                  .set("Name").to(elements[1])
-                  .set("Email").to(elements[2])
-                  .set("Age").to(Integer.valueOf(elements[3]))
-                  .build();
-          c.output(mutation);
-      }
-  }
-}

From 65e740ed2f265b550e70af867535d7bba7306298 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Mon, 15 May 2017 10:16:18 -0700
Subject: [PATCH 321/387] Refine Spanner API tests

And remove outdated Bigtable comment
---
 .../org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
index 8950452b5db2..91caded1ad35 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
@@ -63,7 +63,10 @@ public void testGcpApiSurface() throws Exception {
             Matchers.<Class<?>>equalTo(com.google.cloud.bigtable.grpc.BigtableClusterName.class),
             Matchers.<Class<?>>equalTo(com.google.cloud.bigtable.grpc.BigtableInstanceName.class),
             Matchers.<Class<?>>equalTo(com.google.cloud.bigtable.grpc.BigtableTableName.class),
-            // via Bigtable, PR above out to fix.
+            Matchers.<Class<?>>equalTo(com.google.cloud.ByteArray.class),
+            Matchers.<Class<?>>equalTo(com.google.cloud.Date.class),
+            Matchers.<Class<?>>equalTo(com.google.cloud.Timestamp.class),
+            classesInPackage("com.google.cloud.spanner"),
             classesInPackage("com.google.datastore.v1"),
             classesInPackage("com.google.protobuf"),
             classesInPackage("com.google.type"),
@@ -75,9 +78,6 @@ public void testGcpApiSurface() throws Exception {
             classesInPackage("javax"),
             classesInPackage("org.apache.beam"),
             classesInPackage("org.apache.commons.logging"),
-            classesInPackage("com.google.cloud"),
-            classesInPackage("com.google.cloud.spanner"),
-            // via Bigtable
             classesInPackage("org.joda.time"));
 
     assertThat(apiSurface, containsOnlyClassesMatching(allowedClasses));

From e2070229f82ac4b78c470051cb94b68b58b75f16 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Mon, 15 May 2017 10:42:57 -0700
Subject: [PATCH 322/387] SpannerIO.Write cleanup and style fixes

* Rename to Write to match the rest of the SDK.
* Convert to AutoValue, delete toString.
* Drop .writeTo(), instead use .write() as default constructor.
* Temporarily drop withBatchSize, as its existence is not clearly
  justified.
---
 .../beam/sdk/io/gcp/spanner/SpannerIO.java    | 116 ++++++++++--------
 1 file changed, 62 insertions(+), 54 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
index c9c81a51a384..ec119311c106 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
@@ -19,6 +19,7 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.auto.value.AutoValue;
 import com.google.cloud.spanner.AbortedException;
 import com.google.cloud.spanner.DatabaseClient;
 import com.google.cloud.spanner.DatabaseId;
@@ -26,12 +27,11 @@
 import com.google.cloud.spanner.Spanner;
 import com.google.cloud.spanner.SpannerOptions;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -48,31 +48,29 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * {@link PTransform Transforms} for reading from and writing to
+ * Experimental {@link PTransform Transforms} for reading from and writing to
  * <a href="https://cloud.google.com/spanner">Google Cloud Spanner</a>.
  *
  * <h3>Reading from Cloud Spanner</h3>
- * <strong>Status: Not implemented.</strong>
+ *
+ * <p>This functionality is not yet implemented.
  *
  * <h3>Writing to Cloud Spanner</h3>
- * <strong>Status: Experimental.</strong>
  *
- * <p>{@link SpannerIO#writeTo} batches together and concurrently writes a set of {@link Mutation}s.
- * To configure Cloud Spanner sink, you must apply {@link SpannerIO#writeTo} transform to
- * {@link PCollection} and specify instance and database identifiers. For example, following code
- * sketches out a pipeline that imports data from the CSV file to Cloud Spanner.
+ * <p>The Cloud Spanner {@link SpannerIO.Write} transform writes to Cloud Spanner by executing a
+ * collection of input row {@link Mutation Mutations}. The mutations grouped into batches for
+ * efficiency.
  *
- * <pre>{@code
+ * <p>To configure the write transform, create an instance using {@link #write()} and then specify
+ * the destination Cloud Spanner instance ({@link Write#withInstanceId(String)} and destination
+ * database ({@link Write#withDatabaseId(String)}). For example:
  *
- * Pipeline p = ...;
- * // Read the CSV file.
- * PCollection<String> lines = p.apply("Read CSV file", TextIO.Read.from(options.getInput()));
- * // Parse the line and convert to mutation.
- * PCollection<Mutation> mutations = lines.apply("Parse CSV", parseFromCsv());
+ * <pre>{@code
+ * // Earlier in the pipeline, create a PCollection of Mutations to be written to Cloud Spanner.
+ * PCollection<Mutation> mutations = ...;
  * // Write mutations.
- * mutations.apply("Write", SpannerIO.writeTo(options.getInstanceId(), options.getDatabaseId()));
- * p.run();
- *
+ * mutations.apply(
+ *     "Write", SpannerIO.write().withInstanceId("instance").withDatabaseId("database"));
  * }</pre>
  */
 @Experimental(Experimental.Kind.SOURCE_SINK)
@@ -81,12 +79,14 @@ public class SpannerIO {
   @VisibleForTesting
   static final int SPANNER_MUTATIONS_PER_COMMIT_LIMIT = 20000;
 
-    /**
-     * Creates an instance of {@link Writer}. Use {@link Writer#withBatchSize} to limit the batch
-     * size.
-     */
-  public static Writer writeTo(String instanceId, String databaseId) {
-    return new Writer(instanceId, databaseId, SPANNER_MUTATIONS_PER_COMMIT_LIMIT);
+  /**
+   * Creates an unitialized instance of {@link Write}. Before use, the {@link Write} must be
+   * configured with a {@link Write#withInstanceId} and {@link Write#withDatabaseId} that identify
+   * the Cloud Spanner database being written.
+   */
+  @Experimental
+  public static Write write() {
+    return new AutoValue_SpannerIO_Write.Builder().build();
   }
 
   /**
@@ -95,55 +95,63 @@ public static Writer writeTo(String instanceId, String databaseId) {
    * @see SpannerIO
    */
   @Experimental(Experimental.Kind.SOURCE_SINK)
-  public static class Writer extends PTransform<PCollection<Mutation>, PDone> {
+  @AutoValue
+  public abstract static class Write extends PTransform<PCollection<Mutation>, PDone> {
 
-    private final String instanceId;
-    private final String databaseId;
-    private int batchSize;
+    @Nullable
+    abstract String getInstanceId();
 
-    Writer(String instanceId, String databaseId, int batchSize) {
-      this.instanceId = instanceId;
-      this.databaseId = databaseId;
-      this.batchSize = batchSize;
+    @Nullable
+    abstract String getDatabaseId();
+
+    abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    abstract static class Builder {
+
+      abstract Builder setInstanceId(String instanceId);
+
+      abstract Builder setDatabaseId(String databaseId);
+
+      abstract Write build();
     }
 
     /**
-     * Returns a new {@link Writer} with a limit on the number of mutations per batch.
-     * Defaults to {@link SpannerIO#SPANNER_MUTATIONS_PER_COMMIT_LIMIT}.
+     * Returns a new {@link SpannerIO.Write} that will write to the specified Cloud Spanner
+     * instance.
+     *
+     * <p>Does not modify this object.
      */
-    public Writer withBatchSize(Integer batchSize) {
-      return new Writer(instanceId, databaseId, batchSize);
+    public Write withInstanceId(String instanceId) {
+      return toBuilder().setInstanceId(instanceId).build();
     }
 
-    @Override
-    public PDone expand(PCollection<Mutation> input) {
-      input.apply("Write mutations to Spanner", ParDo.of(
-              new SpannerWriterFn(instanceId, databaseId, batchSize)));
-
-      return PDone.in(input.getPipeline());
+    /**
+     * Returns a new {@link SpannerIO.Write} that will write to the specified Cloud Spanner
+     * database.
+     *
+     * <p>Does not modify this object.
+     */
+    public Write withDatabaseId(String databaseId) {
+      return toBuilder().setDatabaseId(databaseId).build();
     }
 
     @Override
-    public void validate(PipelineOptions options) {
-      checkNotNull(instanceId, "instanceId");
-      checkNotNull(databaseId, "databaseId");
-    }
+    public PDone expand(PCollection<Mutation> input) {
+      input.apply("Write mutations to Spanner",
+          ParDo.of(new SpannerWriterFn(
+              getInstanceId(), getDatabaseId(), SPANNER_MUTATIONS_PER_COMMIT_LIMIT)));
 
-    @Override
-    public String toString() {
-      return MoreObjects.toStringHelper(getClass())
-          .add("instanceId", instanceId)
-          .add("databaseId", databaseId)
-          .toString();
+      return PDone.in(input.getPipeline());
     }
 
     @Override
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
       builder
-          .addIfNotNull(DisplayData.item("instanceId", instanceId)
+          .addIfNotNull(DisplayData.item("instanceId", getInstanceId())
               .withLabel("Output Instance"))
-          .addIfNotNull(DisplayData.item("databaseId", databaseId)
+          .addIfNotNull(DisplayData.item("databaseId", getDatabaseId())
               .withLabel("Output Database"));
     }
   }

From bb2b7064960c458a63c627d96e948f38ef11a898 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Tue, 23 May 2017 12:28:59 -0700
Subject: [PATCH 323/387] Automatically generate Python proto and grpc files.

---
 .gitignore                                    |   1 +
 pom.xml                                       |   2 +-
 sdks/python/MANIFEST.in                       |   1 +
 .../apache_beam/runners/api/__init__.py       |  13 +-
 sdks/python/gen_protos.py                     | 116 ++++++++++++++++++
 sdks/python/run_pylint.sh                     |   1 +
 sdks/python/setup.py                          |  38 +++++-
 sdks/python/tox.ini                           |   3 +
 8 files changed, 157 insertions(+), 18 deletions(-)
 create mode 100644 sdks/python/gen_protos.py

diff --git a/.gitignore b/.gitignore
index 1ecb993777b0..6e077bd1d440 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,7 @@ sdks/python/**/*.egg
 sdks/python/LICENSE
 sdks/python/NOTICE
 sdks/python/README.md
+sdks/python/runners/api/*pb2*.*
 
 # Ignore IntelliJ files.
 .idea/
diff --git a/pom.xml b/pom.xml
index c3a6b73dfb1a..4f833cdb8e49 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1301,7 +1301,7 @@
               <exclude>**/.settings/**/*</exclude>
 
               <!-- Proto/grpc generated wrappers -->
-              <exclude>**/sdks/python/apache_beam/runners/api/*.py</exclude>
+              <exclude>**/apache_beam/runners/api/*_pb2*.py</exclude>
             </excludes>
           </configuration>
         </plugin>
diff --git a/sdks/python/MANIFEST.in b/sdks/python/MANIFEST.in
index 41d80ef1b7ec..c97e57aa1a16 100644
--- a/sdks/python/MANIFEST.in
+++ b/sdks/python/MANIFEST.in
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+include gen_protos.py
 include README.md
 include NOTICE
 include LICENSE
diff --git a/sdks/python/apache_beam/runners/api/__init__.py b/sdks/python/apache_beam/runners/api/__init__.py
index bf95208bd6f9..2750859e4f66 100644
--- a/sdks/python/apache_beam/runners/api/__init__.py
+++ b/sdks/python/apache_beam/runners/api/__init__.py
@@ -17,16 +17,5 @@
 
 """For internal use only; no backwards-compatibility guarantees.
 
-Checked in to avoid protoc dependency for Python development.
-
-Regenerate files with::
-
-    protoc -I../common/runner-api/src/main/proto/ \
-        --python_out=apache_beam/runners/api/ \
-        ../common/runner-api/src/main/proto/*.proto
-
-    protoc -I../common/{fn,runner}-api/src/main/proto/ \
-        --python_out=apache_beam/runners/api/ \
-        --grpc_python_out=apache_beam/runners/api/ \
-        ../common/fn-api/src/main/proto/*.proto
+Automatically generated when running setup.py sdist or build[_py].
 """
diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
new file mode 100644
index 000000000000..edaaff488dfb
--- /dev/null
+++ b/sdks/python/gen_protos.py
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Generates Python proto modules and grpc stubs for Beam protos."""
+
+import glob
+import logging
+import multiprocessing
+import os
+import pkg_resources
+import subprocess
+import sys
+import warnings
+
+
+GRPC_TOOLS = 'grpcio-tools>=1.3.5'
+
+BEAM_PROTO_PATHS = [
+  os.path.join('..', 'common', 'runner-api', 'src', 'main', 'proto'),
+  os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto')
+]
+
+PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'runners', 'api')
+
+
+def generate_proto_files():
+
+  try:
+    import grpc_tools
+  except ImportError:
+    warnings.warn('Installing grpcio-tools is recommended for development.')
+
+  py_sdk_root = os.path.dirname(os.path.abspath(__file__))
+  common = os.path.join(py_sdk_root, '..', 'common')
+  proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS]
+  proto_files = sum(
+      [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
+  out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH)
+  out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]
+
+  if out_files and not proto_files:
+    # We have out_files but no protos; assume they're up to date.
+    # This is actually the common case (e.g. installation from an sdist).
+    return
+
+  elif not out_files and not proto_files:
+    if not common:
+      raise RuntimeError(
+          'Not in apache git tree; unable to find proto definitions.')
+    else:
+      raise RuntimeError(
+          'No proto files found in %s.' % proto_dirs)
+
+  # Regenerate iff the proto files are newer.
+  elif not out_files or len(out_files) < len(proto_files) or (
+      min(os.path.getmtime(path) for path in out_files)
+      <= max(os.path.getmtime(path) for path in proto_files)):
+    try:
+      from grpc_tools import protoc
+    except ImportError:
+      # Use a subprocess to avoid messing with this process' path and imports.
+      # Note that this requires a separate module from setup.py for Windows:
+      # https://docs.python.org/2/library/multiprocessing.html#windows
+      p = multiprocessing.Process(
+          target=_install_grpcio_tools_and_generate_proto_files)
+      p.start()
+      p.join()
+    else:
+      logging.info('Regenerating out-of-date Python proto definitions.')
+      builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto')
+      args = (
+        [sys.executable] +  # expecting to be called from command line
+        ['--proto_path=%s' % builtin_protos] +
+        ['--proto_path=%s' % d for d in proto_dirs] +
+        ['--python_out=%s' % out_dir] +
+        ['--grpc_python_out=%s' % out_dir] +
+        proto_files)
+      ret_code = protoc.main(args)
+      if ret_code:
+        raise RuntimeError(
+            'Protoc returned non-zero status (see logs for details): '
+            '%s' % ret_code)
+
+
+# Though wheels are available for grpcio-tools, setup_requires uses
+# easy_install which doesn't understand them.  This means that it is
+# compiled from scratch (which is expensive as it compiles the full
+# protoc compiler).  Instead, we attempt to install a wheel in a temporary
+# directory and add it to the path as needed.
+# See https://github.com/pypa/setuptools/issues/377
+def _install_grpcio_tools_and_generate_proto_files():
+  install_path = os.path.join(
+      os.path.dirname(os.path.abspath(__file__)), '.eggs', 'grpcio-wheels')
+  logging.warning('Downloading a grpcio-tools to %s' % install_path)
+  subprocess.check_call(
+      ['pip', 'install', '-t', install_path, '--upgrade', GRPC_TOOLS])
+  sys.path.append(install_path)
+  generate_proto_files()
+
+
+if __name__ == '__main__':
+  generate_proto_files()
\ No newline at end of file
diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh
index a5e3fa147a81..400c577519dd 100755
--- a/sdks/python/run_pylint.sh
+++ b/sdks/python/run_pylint.sh
@@ -49,6 +49,7 @@ EXCLUDED_GENERATED_FILES=(
 "apache_beam/runners/api/beam_fn_api_pb2.py"
 "apache_beam/runners/api/beam_fn_api_pb2_grpc.py"
 "apache_beam/runners/api/beam_runner_api_pb2.py"
+"apache_beam/runners/api/beam_runner_api_pb2_grpc.py"
 )
 
 FILES_TO_IGNORE=""
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index befc0249c0bc..8a8ce482742a 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -19,15 +19,25 @@
 
 from distutils.version import StrictVersion
 
+import glob
 import os
+import pkg_resources
 import platform
 import shutil
+import subprocess
+import sys
 import warnings
 
 import setuptools
 
+from setuptools.command.build_py import build_py
+from setuptools.command.sdist import sdist
+from setuptools.command.test import test
+
 from pkg_resources import get_distribution, DistributionNotFound
 
+import gen_protos
+
 
 def get_version():
   global_names = {}
@@ -98,8 +108,12 @@ def get_version():
     'pyyaml>=3.12,<4.0.0',
     ]
 
+REQUIRED_SETUP_PACKAGES = [
+    'nose>=1.0',
+    ]
+
 REQUIRED_TEST_PACKAGES = [
-    'pyhamcrest>=1.9,<2.0'
+    'pyhamcrest>=1.9,<2.0',
     ]
 
 GCP_REQUIREMENTS = [
@@ -111,6 +125,15 @@ def get_version():
 ]
 
 
+# We must generate protos after setup_requires are installed.
+def generate_protos_first(original_cmd):
+  class cmd(original_cmd, object):
+    def run(self):
+      gen_protos.generate_proto_files()
+      super(cmd, self).run()
+  return cmd
+
+
 setuptools.setup(
     name=PACKAGE_NAME,
     version=PACKAGE_VERSION,
@@ -135,7 +158,7 @@ def get_version():
         'apache_beam/utils/counters.py',
         'apache_beam/utils/windowed_value.py',
     ]),
-    setup_requires=['nose>=1.0'],
+    setup_requires=REQUIRED_SETUP_PACKAGES,
     install_requires=REQUIRED_PACKAGES,
     test_suite='nose.collector',
     tests_require=REQUIRED_TEST_PACKAGES,
@@ -153,11 +176,16 @@ def get_version():
         'Programming Language :: Python :: 2.7',
         'Topic :: Software Development :: Libraries',
         'Topic :: Software Development :: Libraries :: Python Modules',
-        ],
+    ],
     license='Apache License, Version 2.0',
     keywords=PACKAGE_KEYWORDS,
     entry_points={
         'nose.plugins.0.10': [
             'beam_test_plugin = test_config:BeamTestPlugin'
-            ]}
-    )
+    ]},
+    cmdclass={
+        'build_py': generate_protos_first(build_py),
+        'sdist': generate_protos_first(sdist),
+        'test': generate_protos_first(test),
+    },
+)
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 2592b17501ae..917e907b5ce2 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -28,6 +28,7 @@ select = E3
 # autocomplete_test depends on nose when invoked directly.
 deps =
   nose==1.3.7
+  grpcio-tools==1.3.5
 commands =
   python --version
   pip install -e .[test]
@@ -44,6 +45,7 @@ platform = linux2
 # autocomplete_test depends on nose when invoked directly.
 deps =
   nose==1.3.7
+  grpcio-tools==1.3.5
   cython==0.25.2
 whitelist_externals=find
 commands =
@@ -87,6 +89,7 @@ passenv = TRAVIS*
 [testenv:docs]
 deps=
   nose==1.3.7
+  grpcio-tools==1.3.5
   Sphinx==1.5.5
 commands =
   pip install -e .[test,gcp,docs]

From c2e85daa68b444a9daeb582e686400bdee405964 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Tue, 23 May 2017 12:30:30 -0700
Subject: [PATCH 324/387] Remove auto-generated proto and grpc files.

---
 .../runners/api/beam_fn_api_pb2.py            | 2742 ----------------
 .../runners/api/beam_fn_api_pb2_grpc.py       |  205 --
 .../runners/api/beam_runner_api_pb2.py        | 2872 -----------------
 3 files changed, 5819 deletions(-)
 delete mode 100644 sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
 delete mode 100644 sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
 delete mode 100644 sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py

diff --git a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
deleted file mode 100644
index cb0b72b7d1fc..000000000000
--- a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2.py
+++ /dev/null
@@ -1,2742 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: beam_fn_api.proto
-
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
-from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
-
-# This module is experimental. No backwards-compatibility guarantees.
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
-  name='beam_fn_api.proto',
-  package='org.apache.beam.fn.v1',
-  syntax='proto3',
-  serialized_pb=_b('\n\x11\x62\x65\x61m_fn_api.proto\x12\x15org.apache.beam.fn.v1\x1a\x19google/protobuf/any.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"t\n\x06Target\x12%\n\x1dprimitive_transform_reference\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x1a\x35\n\x04List\x12-\n\x06target\x18\x01 \x03(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\"&\n\x0bPCollection\x12\x17\n\x0f\x63oder_reference\x18\x01 \x01(\t\"\xcb\x04\n\x12PrimitiveTransform\x12\n\n\x02id\x18\x01 \x01(\t\x12:\n\rfunction_spec\x18\x02 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\x12\x45\n\x06inputs\x18\x03 \x03(\x0b\x32\x35.org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry\x12G\n\x07outputs\x18\x04 \x03(\x0b\x32\x36.org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry\x12N\n\x0bside_inputs\x18\x05 \x03(\x0b\x32\x39.org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry\x12\x11\n\tstep_name\x18\x06 \x01(\t\x1aQ\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".org.apache.beam.fn.v1.Target.List:\x02\x38\x01\x1aR\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".org.apache.beam.fn.v1.PCollection:\x02\x38\x01\x1aS\n\x0fSideInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .org.apache.beam.fn.v1.SideInput:\x02\x38\x01\"j\n\x0c\x46unctionSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0b\n\x03urn\x18\x02 \x01(\t\x12\x1d\n\x15\x65nvironment_reference\x18\x03 \x01(\t\x12\"\n\x04\x64\x61ta\x18\x04 \x01(\x0b\x32\x14.google.protobuf.Any\"o\n\tSideInput\x12,\n\x05input\x18\x01 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\x34\n\x07view_fn\x18\x02 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\"f\n\x05\x43oder\x12:\n\rfunction_spec\x18\x01 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\x12!\n\x19\x63omponent_coder_reference\x18\x02 \x03(\t\"]\n\x0eRemoteGrpcPort\x12K\n\x16\x61pi_service_descriptor\x18\x01 \x01(\x0b\x32+.org.apache.beam.fn.v1.ApiServiceDescriptor\"\xe8\x02\n\x12InstructionRequest\x12\x16\n\x0einstruction_id\x18\x01 \x01(\t\x12;\n\x08register\x18\xe8\x07 \x01(\x0b\x32&.org.apache.beam.fn.v1.RegisterRequestH\x00\x12\x46\n\x0eprocess_bundle\x18\xe9\x07 \x01(\x0b\x32+.org.apache.beam.fn.v1.ProcessBundleRequestH\x00\x12W\n\x17process_bundle_progress\x18\xea\x07 \x01(\x0b\x32\x33.org.apache.beam.fn.v1.ProcessBundleProgressRequestH\x00\x12Q\n\x14process_bundle_split\x18\xeb\x07 \x01(\x0b\x32\x30.org.apache.beam.fn.v1.ProcessBundleSplitRequestH\x00\x42\t\n\x07request\"\xfd\x02\n\x13InstructionResponse\x12\x16\n\x0einstruction_id\x18\x01 \x01(\t\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12<\n\x08register\x18\xe8\x07 \x01(\x0b\x32\'.org.apache.beam.fn.v1.RegisterResponseH\x00\x12G\n\x0eprocess_bundle\x18\xe9\x07 \x01(\x0b\x32,.org.apache.beam.fn.v1.ProcessBundleResponseH\x00\x12X\n\x17process_bundle_progress\x18\xea\x07 \x01(\x0b\x32\x34.org.apache.beam.fn.v1.ProcessBundleProgressResponseH\x00\x12R\n\x14process_bundle_split\x18\xeb\x07 \x01(\x0b\x32\x31.org.apache.beam.fn.v1.ProcessBundleSplitResponseH\x00\x42\n\n\x08response\"d\n\x0fRegisterRequest\x12Q\n\x19process_bundle_descriptor\x18\x01 \x03(\x0b\x32..org.apache.beam.fn.v1.ProcessBundleDescriptor\"\x12\n\x10RegisterResponse\"\x9b\x01\n\x17ProcessBundleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\x46\n\x13primitive_transform\x18\x02 \x03(\x0b\x32).org.apache.beam.fn.v1.PrimitiveTransform\x12,\n\x06\x63oders\x18\x04 \x03(\x0b\x32\x1c.org.apache.beam.fn.v1.Coder\"|\n\x14ProcessBundleRequest\x12+\n#process_bundle_descriptor_reference\x18\x01 \x01(\t\x12\x37\n\x0c\x63\x61\x63he_tokens\x18\x02 \x03(\x0b\x32!.org.apache.beam.fn.v1.CacheToken\"\x17\n\x15ProcessBundleResponse\"=\n\x1cProcessBundleProgressRequest\x12\x1d\n\x15instruction_reference\x18\x01 \x01(\t\"G\n\x1dProcessBundleProgressResponse\x12\x15\n\rfinished_work\x18\x01 \x01(\x01\x12\x0f\n\x07\x62\x61\x63klog\x18\x02 \x01(\x01\"L\n\x19ProcessBundleSplitRequest\x12\x1d\n\x15instruction_reference\x18\x01 \x01(\t\x12\x10\n\x08\x66raction\x18\x02 \x01(\x01\"(\n\x17\x45lementCountRestriction\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\",\n\x1b\x45lementCountSkipRestriction\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\"\xc8\x01\n\x17PrimitiveTransformSplit\x12%\n\x1dprimitive_transform_reference\x18\x01 \x01(\t\x12\x42\n\x15\x63ompleted_restriction\x18\x02 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\x12\x42\n\x15remaining_restriction\x18\x03 \x01(\x0b\x32#.org.apache.beam.fn.v1.FunctionSpec\"\\\n\x1aProcessBundleSplitResponse\x12>\n\x06splits\x18\x01 \x03(\x0b\x32..org.apache.beam.fn.v1.PrimitiveTransformSplit\"\xa2\x01\n\x08\x45lements\x12\x32\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32$.org.apache.beam.fn.v1.Elements.Data\x1a\x62\n\x04\x44\x61ta\x12\x1d\n\x15instruction_reference\x18\x01 \x01(\t\x12-\n\x06target\x18\x02 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xaa\x02\n\x0cStateRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1d\n\x15instruction_reference\x18\x02 \x01(\t\x12\x32\n\tstate_key\x18\x03 \x01(\x0b\x32\x1f.org.apache.beam.fn.v1.StateKey\x12\x36\n\x03get\x18\xe8\x07 \x01(\x0b\x32&.org.apache.beam.fn.v1.StateGetRequestH\x00\x12<\n\x06\x61ppend\x18\xe9\x07 \x01(\x0b\x32).org.apache.beam.fn.v1.StateAppendRequestH\x00\x12:\n\x05\x63lear\x18\xea\x07 \x01(\x0b\x32(.org.apache.beam.fn.v1.StateClearRequestH\x00\x42\t\n\x07request\"\xeb\x01\n\rStateResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x37\n\x03get\x18\xe8\x07 \x01(\x0b\x32\'.org.apache.beam.fn.v1.StateGetResponseH\x00\x12=\n\x06\x61ppend\x18\xe9\x07 \x01(\x0b\x32*.org.apache.beam.fn.v1.StateAppendResponseH\x00\x12;\n\x05\x63lear\x18\xea\x07 \x01(\x0b\x32).org.apache.beam.fn.v1.StateClearResponseH\x00\x42\n\n\x08response\"J\n\nCacheToken\x12-\n\x06target\x18\x01 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\r\n\x05token\x18\x02 \x01(\x0c\"V\n\x08StateKey\x12-\n\x06target\x18\x01 \x01(\x0b\x32\x1d.org.apache.beam.fn.v1.Target\x12\x0e\n\x06window\x18\x02 \x01(\x0c\x12\x0b\n\x03key\x18\x03 \x01(\x0c\"=\n\x11\x43ontinuableStream\x12\x1a\n\x12\x63ontinuation_token\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"-\n\x0fStateGetRequest\x12\x1a\n\x12\x63ontinuation_token\x18\x01 \x01(\x0c\"L\n\x10StateGetResponse\x12\x38\n\x06stream\x18\x01 \x01(\x0b\x32(.org.apache.beam.fn.v1.ContinuableStream\"\"\n\x12StateAppendRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"\x15\n\x13StateAppendResponse\"\x13\n\x11StateClearRequest\"\x14\n\x12StateClearResponse\"\x9a\x03\n\x08LogEntry\x12:\n\x08severity\x18\x01 \x01(\x0e\x32(.org.apache.beam.fn.v1.LogEntry.Severity\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07message\x18\x03 \x01(\t\x12\r\n\x05trace\x18\x04 \x01(\t\x12\x1d\n\x15instruction_reference\x18\x05 \x01(\t\x12%\n\x1dprimitive_transform_reference\x18\x06 \x01(\t\x12\x14\n\x0clog_location\x18\x07 \x01(\t\x12\x0e\n\x06thread\x18\x08 \x01(\t\x1a<\n\x04List\x12\x34\n\x0blog_entries\x18\x01 \x03(\x0b\x32\x1f.org.apache.beam.fn.v1.LogEntry\"Y\n\x08Severity\x12\t\n\x05TRACE\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\n\x12\x08\n\x04INFO\x10\x14\x12\n\n\x06NOTICE\x10\x1e\x12\x08\n\x04WARN\x10(\x12\t\n\x05\x45RROR\x10\x32\x12\x0c\n\x08\x43RITICAL\x10<\"\x0c\n\nLogControl\"\xa1\x01\n\x14\x41piServiceDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12^\n\x1foauth2_client_credentials_grant\x18\x03 \x01(\x0b\x32\x33.org.apache.beam.fn.v1.OAuth2ClientCredentialsGrantH\x00\x42\x10\n\x0e\x61uthentication\"+\n\x1cOAuth2ClientCredentialsGrant\x12\x0b\n\x03url\x18\x01 \x01(\t\"F\n\x0f\x44ockerContainer\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1a\n\x12registry_reference\x18\x03 \x01(\t2w\n\rBeamFnControl\x12\x66\n\x07\x43ontrol\x12*.org.apache.beam.fn.v1.InstructionResponse\x1a).org.apache.beam.fn.v1.InstructionRequest\"\x00(\x01\x30\x01\x32\\\n\nBeamFnData\x12N\n\x04\x44\x61ta\x12\x1f.org.apache.beam.fn.v1.Elements\x1a\x1f.org.apache.beam.fn.v1.Elements\"\x00(\x01\x30\x01\x32g\n\x0b\x42\x65\x61mFnState\x12X\n\x05State\x12#.org.apache.beam.fn.v1.StateRequest\x1a$.org.apache.beam.fn.v1.StateResponse\"\x00(\x01\x30\x01\x32i\n\rBeamFnLogging\x12X\n\x07Logging\x12$.org.apache.beam.fn.v1.LogEntry.List\x1a!.org.apache.beam.fn.v1.LogControl\"\x00(\x01\x30\x01\x42\"\n\x15org.apache.beam.fn.v1B\tBeamFnApib\x06proto3')
-  ,
-  dependencies=[google_dot_protobuf_dot_any__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-
-
-_LOGENTRY_SEVERITY = _descriptor.EnumDescriptor(
-  name='Severity',
-  full_name='org.apache.beam.fn.v1.LogEntry.Severity',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='TRACE', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='DEBUG', index=1, number=10,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='INFO', index=2, number=20,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='NOTICE', index=3, number=30,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='WARN', index=4, number=40,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='ERROR', index=5, number=50,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='CRITICAL', index=6, number=60,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=4529,
-  serialized_end=4618,
-)
-_sym_db.RegisterEnumDescriptor(_LOGENTRY_SEVERITY)
-
-
-_TARGET_LIST = _descriptor.Descriptor(
-  name='List',
-  full_name='org.apache.beam.fn.v1.Target.List',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='target', full_name='org.apache.beam.fn.v1.Target.List.target', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=167,
-  serialized_end=220,
-)
-
-_TARGET = _descriptor.Descriptor(
-  name='Target',
-  full_name='org.apache.beam.fn.v1.Target',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='primitive_transform_reference', full_name='org.apache.beam.fn.v1.Target.primitive_transform_reference', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='name', full_name='org.apache.beam.fn.v1.Target.name', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_TARGET_LIST, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=104,
-  serialized_end=220,
-)
-
-
-_PCOLLECTION = _descriptor.Descriptor(
-  name='PCollection',
-  full_name='org.apache.beam.fn.v1.PCollection',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='coder_reference', full_name='org.apache.beam.fn.v1.PCollection.coder_reference', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=222,
-  serialized_end=260,
-)
-
-
-_PRIMITIVETRANSFORM_INPUTSENTRY = _descriptor.Descriptor(
-  name='InputsEntry',
-  full_name='org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=600,
-  serialized_end=681,
-)
-
-_PRIMITIVETRANSFORM_OUTPUTSENTRY = _descriptor.Descriptor(
-  name='OutputsEntry',
-  full_name='org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=683,
-  serialized_end=765,
-)
-
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY = _descriptor.Descriptor(
-  name='SideInputsEntry',
-  full_name='org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=767,
-  serialized_end=850,
-)
-
-_PRIMITIVETRANSFORM = _descriptor.Descriptor(
-  name='PrimitiveTransform',
-  full_name='org.apache.beam.fn.v1.PrimitiveTransform',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.fn.v1.PrimitiveTransform.id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='function_spec', full_name='org.apache.beam.fn.v1.PrimitiveTransform.function_spec', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='inputs', full_name='org.apache.beam.fn.v1.PrimitiveTransform.inputs', index=2,
-      number=3, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='outputs', full_name='org.apache.beam.fn.v1.PrimitiveTransform.outputs', index=3,
-      number=4, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='side_inputs', full_name='org.apache.beam.fn.v1.PrimitiveTransform.side_inputs', index=4,
-      number=5, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='step_name', full_name='org.apache.beam.fn.v1.PrimitiveTransform.step_name', index=5,
-      number=6, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_PRIMITIVETRANSFORM_INPUTSENTRY, _PRIMITIVETRANSFORM_OUTPUTSENTRY, _PRIMITIVETRANSFORM_SIDEINPUTSENTRY, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=263,
-  serialized_end=850,
-)
-
-
-_FUNCTIONSPEC = _descriptor.Descriptor(
-  name='FunctionSpec',
-  full_name='org.apache.beam.fn.v1.FunctionSpec',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.fn.v1.FunctionSpec.id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='urn', full_name='org.apache.beam.fn.v1.FunctionSpec.urn', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='environment_reference', full_name='org.apache.beam.fn.v1.FunctionSpec.environment_reference', index=2,
-      number=3, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='data', full_name='org.apache.beam.fn.v1.FunctionSpec.data', index=3,
-      number=4, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=852,
-  serialized_end=958,
-)
-
-
-_SIDEINPUT = _descriptor.Descriptor(
-  name='SideInput',
-  full_name='org.apache.beam.fn.v1.SideInput',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='input', full_name='org.apache.beam.fn.v1.SideInput.input', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='view_fn', full_name='org.apache.beam.fn.v1.SideInput.view_fn', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=960,
-  serialized_end=1071,
-)
-
-
-_CODER = _descriptor.Descriptor(
-  name='Coder',
-  full_name='org.apache.beam.fn.v1.Coder',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='function_spec', full_name='org.apache.beam.fn.v1.Coder.function_spec', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='component_coder_reference', full_name='org.apache.beam.fn.v1.Coder.component_coder_reference', index=1,
-      number=2, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=1073,
-  serialized_end=1175,
-)
-
-
-_REMOTEGRPCPORT = _descriptor.Descriptor(
-  name='RemoteGrpcPort',
-  full_name='org.apache.beam.fn.v1.RemoteGrpcPort',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='api_service_descriptor', full_name='org.apache.beam.fn.v1.RemoteGrpcPort.api_service_descriptor', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=1177,
-  serialized_end=1270,
-)
-
-
-_INSTRUCTIONREQUEST = _descriptor.Descriptor(
-  name='InstructionRequest',
-  full_name='org.apache.beam.fn.v1.InstructionRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='instruction_id', full_name='org.apache.beam.fn.v1.InstructionRequest.instruction_id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='register', full_name='org.apache.beam.fn.v1.InstructionRequest.register', index=1,
-      number=1000, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='process_bundle', full_name='org.apache.beam.fn.v1.InstructionRequest.process_bundle', index=2,
-      number=1001, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='process_bundle_progress', full_name='org.apache.beam.fn.v1.InstructionRequest.process_bundle_progress', index=3,
-      number=1002, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='process_bundle_split', full_name='org.apache.beam.fn.v1.InstructionRequest.process_bundle_split', index=4,
-      number=1003, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='request', full_name='org.apache.beam.fn.v1.InstructionRequest.request',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=1273,
-  serialized_end=1633,
-)
-
-
-_INSTRUCTIONRESPONSE = _descriptor.Descriptor(
-  name='InstructionResponse',
-  full_name='org.apache.beam.fn.v1.InstructionResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='instruction_id', full_name='org.apache.beam.fn.v1.InstructionResponse.instruction_id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='error', full_name='org.apache.beam.fn.v1.InstructionResponse.error', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='register', full_name='org.apache.beam.fn.v1.InstructionResponse.register', index=2,
-      number=1000, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='process_bundle', full_name='org.apache.beam.fn.v1.InstructionResponse.process_bundle', index=3,
-      number=1001, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='process_bundle_progress', full_name='org.apache.beam.fn.v1.InstructionResponse.process_bundle_progress', index=4,
-      number=1002, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='process_bundle_split', full_name='org.apache.beam.fn.v1.InstructionResponse.process_bundle_split', index=5,
-      number=1003, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='response', full_name='org.apache.beam.fn.v1.InstructionResponse.response',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=1636,
-  serialized_end=2017,
-)
-
-
-_REGISTERREQUEST = _descriptor.Descriptor(
-  name='RegisterRequest',
-  full_name='org.apache.beam.fn.v1.RegisterRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='process_bundle_descriptor', full_name='org.apache.beam.fn.v1.RegisterRequest.process_bundle_descriptor', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2019,
-  serialized_end=2119,
-)
-
-
-_REGISTERRESPONSE = _descriptor.Descriptor(
-  name='RegisterResponse',
-  full_name='org.apache.beam.fn.v1.RegisterResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2121,
-  serialized_end=2139,
-)
-
-
-_PROCESSBUNDLEDESCRIPTOR = _descriptor.Descriptor(
-  name='ProcessBundleDescriptor',
-  full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor.id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='primitive_transform', full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor.primitive_transform', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='coders', full_name='org.apache.beam.fn.v1.ProcessBundleDescriptor.coders', index=2,
-      number=4, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2142,
-  serialized_end=2297,
-)
-
-
-_PROCESSBUNDLEREQUEST = _descriptor.Descriptor(
-  name='ProcessBundleRequest',
-  full_name='org.apache.beam.fn.v1.ProcessBundleRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='process_bundle_descriptor_reference', full_name='org.apache.beam.fn.v1.ProcessBundleRequest.process_bundle_descriptor_reference', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='cache_tokens', full_name='org.apache.beam.fn.v1.ProcessBundleRequest.cache_tokens', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2299,
-  serialized_end=2423,
-)
-
-
-_PROCESSBUNDLERESPONSE = _descriptor.Descriptor(
-  name='ProcessBundleResponse',
-  full_name='org.apache.beam.fn.v1.ProcessBundleResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2425,
-  serialized_end=2448,
-)
-
-
-_PROCESSBUNDLEPROGRESSREQUEST = _descriptor.Descriptor(
-  name='ProcessBundleProgressRequest',
-  full_name='org.apache.beam.fn.v1.ProcessBundleProgressRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='instruction_reference', full_name='org.apache.beam.fn.v1.ProcessBundleProgressRequest.instruction_reference', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2450,
-  serialized_end=2511,
-)
-
-
-_PROCESSBUNDLEPROGRESSRESPONSE = _descriptor.Descriptor(
-  name='ProcessBundleProgressResponse',
-  full_name='org.apache.beam.fn.v1.ProcessBundleProgressResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='finished_work', full_name='org.apache.beam.fn.v1.ProcessBundleProgressResponse.finished_work', index=0,
-      number=1, type=1, cpp_type=5, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='backlog', full_name='org.apache.beam.fn.v1.ProcessBundleProgressResponse.backlog', index=1,
-      number=2, type=1, cpp_type=5, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2513,
-  serialized_end=2584,
-)
-
-
-_PROCESSBUNDLESPLITREQUEST = _descriptor.Descriptor(
-  name='ProcessBundleSplitRequest',
-  full_name='org.apache.beam.fn.v1.ProcessBundleSplitRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='instruction_reference', full_name='org.apache.beam.fn.v1.ProcessBundleSplitRequest.instruction_reference', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='fraction', full_name='org.apache.beam.fn.v1.ProcessBundleSplitRequest.fraction', index=1,
-      number=2, type=1, cpp_type=5, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2586,
-  serialized_end=2662,
-)
-
-
-_ELEMENTCOUNTRESTRICTION = _descriptor.Descriptor(
-  name='ElementCountRestriction',
-  full_name='org.apache.beam.fn.v1.ElementCountRestriction',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='count', full_name='org.apache.beam.fn.v1.ElementCountRestriction.count', index=0,
-      number=1, type=3, cpp_type=2, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2664,
-  serialized_end=2704,
-)
-
-
-_ELEMENTCOUNTSKIPRESTRICTION = _descriptor.Descriptor(
-  name='ElementCountSkipRestriction',
-  full_name='org.apache.beam.fn.v1.ElementCountSkipRestriction',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='count', full_name='org.apache.beam.fn.v1.ElementCountSkipRestriction.count', index=0,
-      number=1, type=3, cpp_type=2, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2706,
-  serialized_end=2750,
-)
-
-
-_PRIMITIVETRANSFORMSPLIT = _descriptor.Descriptor(
-  name='PrimitiveTransformSplit',
-  full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='primitive_transform_reference', full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit.primitive_transform_reference', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='completed_restriction', full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit.completed_restriction', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='remaining_restriction', full_name='org.apache.beam.fn.v1.PrimitiveTransformSplit.remaining_restriction', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2753,
-  serialized_end=2953,
-)
-
-
-_PROCESSBUNDLESPLITRESPONSE = _descriptor.Descriptor(
-  name='ProcessBundleSplitResponse',
-  full_name='org.apache.beam.fn.v1.ProcessBundleSplitResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='splits', full_name='org.apache.beam.fn.v1.ProcessBundleSplitResponse.splits', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2955,
-  serialized_end=3047,
-)
-
-
-_ELEMENTS_DATA = _descriptor.Descriptor(
-  name='Data',
-  full_name='org.apache.beam.fn.v1.Elements.Data',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='instruction_reference', full_name='org.apache.beam.fn.v1.Elements.Data.instruction_reference', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='target', full_name='org.apache.beam.fn.v1.Elements.Data.target', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='data', full_name='org.apache.beam.fn.v1.Elements.Data.data', index=2,
-      number=3, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3114,
-  serialized_end=3212,
-)
-
-_ELEMENTS = _descriptor.Descriptor(
-  name='Elements',
-  full_name='org.apache.beam.fn.v1.Elements',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='data', full_name='org.apache.beam.fn.v1.Elements.data', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_ELEMENTS_DATA, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3050,
-  serialized_end=3212,
-)
-
-
-_STATEREQUEST = _descriptor.Descriptor(
-  name='StateRequest',
-  full_name='org.apache.beam.fn.v1.StateRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.fn.v1.StateRequest.id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='instruction_reference', full_name='org.apache.beam.fn.v1.StateRequest.instruction_reference', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='state_key', full_name='org.apache.beam.fn.v1.StateRequest.state_key', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='get', full_name='org.apache.beam.fn.v1.StateRequest.get', index=3,
-      number=1000, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='append', full_name='org.apache.beam.fn.v1.StateRequest.append', index=4,
-      number=1001, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='clear', full_name='org.apache.beam.fn.v1.StateRequest.clear', index=5,
-      number=1002, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='request', full_name='org.apache.beam.fn.v1.StateRequest.request',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=3215,
-  serialized_end=3513,
-)
-
-
-_STATERESPONSE = _descriptor.Descriptor(
-  name='StateResponse',
-  full_name='org.apache.beam.fn.v1.StateResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.fn.v1.StateResponse.id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='error', full_name='org.apache.beam.fn.v1.StateResponse.error', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='get', full_name='org.apache.beam.fn.v1.StateResponse.get', index=2,
-      number=1000, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='append', full_name='org.apache.beam.fn.v1.StateResponse.append', index=3,
-      number=1001, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='clear', full_name='org.apache.beam.fn.v1.StateResponse.clear', index=4,
-      number=1002, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='response', full_name='org.apache.beam.fn.v1.StateResponse.response',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=3516,
-  serialized_end=3751,
-)
-
-
-_CACHETOKEN = _descriptor.Descriptor(
-  name='CacheToken',
-  full_name='org.apache.beam.fn.v1.CacheToken',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='target', full_name='org.apache.beam.fn.v1.CacheToken.target', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='token', full_name='org.apache.beam.fn.v1.CacheToken.token', index=1,
-      number=2, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3753,
-  serialized_end=3827,
-)
-
-
-_STATEKEY = _descriptor.Descriptor(
-  name='StateKey',
-  full_name='org.apache.beam.fn.v1.StateKey',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='target', full_name='org.apache.beam.fn.v1.StateKey.target', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='window', full_name='org.apache.beam.fn.v1.StateKey.window', index=1,
-      number=2, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.fn.v1.StateKey.key', index=2,
-      number=3, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3829,
-  serialized_end=3915,
-)
-
-
-_CONTINUABLESTREAM = _descriptor.Descriptor(
-  name='ContinuableStream',
-  full_name='org.apache.beam.fn.v1.ContinuableStream',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='continuation_token', full_name='org.apache.beam.fn.v1.ContinuableStream.continuation_token', index=0,
-      number=1, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='data', full_name='org.apache.beam.fn.v1.ContinuableStream.data', index=1,
-      number=2, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3917,
-  serialized_end=3978,
-)
-
-
-_STATEGETREQUEST = _descriptor.Descriptor(
-  name='StateGetRequest',
-  full_name='org.apache.beam.fn.v1.StateGetRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='continuation_token', full_name='org.apache.beam.fn.v1.StateGetRequest.continuation_token', index=0,
-      number=1, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3980,
-  serialized_end=4025,
-)
-
-
-_STATEGETRESPONSE = _descriptor.Descriptor(
-  name='StateGetResponse',
-  full_name='org.apache.beam.fn.v1.StateGetResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='stream', full_name='org.apache.beam.fn.v1.StateGetResponse.stream', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4027,
-  serialized_end=4103,
-)
-
-
-_STATEAPPENDREQUEST = _descriptor.Descriptor(
-  name='StateAppendRequest',
-  full_name='org.apache.beam.fn.v1.StateAppendRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='data', full_name='org.apache.beam.fn.v1.StateAppendRequest.data', index=0,
-      number=1, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4105,
-  serialized_end=4139,
-)
-
-
-_STATEAPPENDRESPONSE = _descriptor.Descriptor(
-  name='StateAppendResponse',
-  full_name='org.apache.beam.fn.v1.StateAppendResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4141,
-  serialized_end=4162,
-)
-
-
-_STATECLEARREQUEST = _descriptor.Descriptor(
-  name='StateClearRequest',
-  full_name='org.apache.beam.fn.v1.StateClearRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4164,
-  serialized_end=4183,
-)
-
-
-_STATECLEARRESPONSE = _descriptor.Descriptor(
-  name='StateClearResponse',
-  full_name='org.apache.beam.fn.v1.StateClearResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4185,
-  serialized_end=4205,
-)
-
-
-_LOGENTRY_LIST = _descriptor.Descriptor(
-  name='List',
-  full_name='org.apache.beam.fn.v1.LogEntry.List',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='log_entries', full_name='org.apache.beam.fn.v1.LogEntry.List.log_entries', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4467,
-  serialized_end=4527,
-)
-
-_LOGENTRY = _descriptor.Descriptor(
-  name='LogEntry',
-  full_name='org.apache.beam.fn.v1.LogEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='severity', full_name='org.apache.beam.fn.v1.LogEntry.severity', index=0,
-      number=1, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='timestamp', full_name='org.apache.beam.fn.v1.LogEntry.timestamp', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='message', full_name='org.apache.beam.fn.v1.LogEntry.message', index=2,
-      number=3, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='trace', full_name='org.apache.beam.fn.v1.LogEntry.trace', index=3,
-      number=4, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='instruction_reference', full_name='org.apache.beam.fn.v1.LogEntry.instruction_reference', index=4,
-      number=5, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='primitive_transform_reference', full_name='org.apache.beam.fn.v1.LogEntry.primitive_transform_reference', index=5,
-      number=6, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='log_location', full_name='org.apache.beam.fn.v1.LogEntry.log_location', index=6,
-      number=7, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='thread', full_name='org.apache.beam.fn.v1.LogEntry.thread', index=7,
-      number=8, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_LOGENTRY_LIST, ],
-  enum_types=[
-    _LOGENTRY_SEVERITY,
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4208,
-  serialized_end=4618,
-)
-
-
-_LOGCONTROL = _descriptor.Descriptor(
-  name='LogControl',
-  full_name='org.apache.beam.fn.v1.LogControl',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4620,
-  serialized_end=4632,
-)
-
-
-_APISERVICEDESCRIPTOR = _descriptor.Descriptor(
-  name='ApiServiceDescriptor',
-  full_name='org.apache.beam.fn.v1.ApiServiceDescriptor',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='url', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.url', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='oauth2_client_credentials_grant', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.oauth2_client_credentials_grant', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='authentication', full_name='org.apache.beam.fn.v1.ApiServiceDescriptor.authentication',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=4635,
-  serialized_end=4796,
-)
-
-
-_OAUTH2CLIENTCREDENTIALSGRANT = _descriptor.Descriptor(
-  name='OAuth2ClientCredentialsGrant',
-  full_name='org.apache.beam.fn.v1.OAuth2ClientCredentialsGrant',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='url', full_name='org.apache.beam.fn.v1.OAuth2ClientCredentialsGrant.url', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4798,
-  serialized_end=4841,
-)
-
-
-_DOCKERCONTAINER = _descriptor.Descriptor(
-  name='DockerContainer',
-  full_name='org.apache.beam.fn.v1.DockerContainer',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.fn.v1.DockerContainer.id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='uri', full_name='org.apache.beam.fn.v1.DockerContainer.uri', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='registry_reference', full_name='org.apache.beam.fn.v1.DockerContainer.registry_reference', index=2,
-      number=3, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4843,
-  serialized_end=4913,
-)
-
-_TARGET_LIST.fields_by_name['target'].message_type = _TARGET
-_TARGET_LIST.containing_type = _TARGET
-_PRIMITIVETRANSFORM_INPUTSENTRY.fields_by_name['value'].message_type = _TARGET_LIST
-_PRIMITIVETRANSFORM_INPUTSENTRY.containing_type = _PRIMITIVETRANSFORM
-_PRIMITIVETRANSFORM_OUTPUTSENTRY.fields_by_name['value'].message_type = _PCOLLECTION
-_PRIMITIVETRANSFORM_OUTPUTSENTRY.containing_type = _PRIMITIVETRANSFORM
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY.fields_by_name['value'].message_type = _SIDEINPUT
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY.containing_type = _PRIMITIVETRANSFORM
-_PRIMITIVETRANSFORM.fields_by_name['function_spec'].message_type = _FUNCTIONSPEC
-_PRIMITIVETRANSFORM.fields_by_name['inputs'].message_type = _PRIMITIVETRANSFORM_INPUTSENTRY
-_PRIMITIVETRANSFORM.fields_by_name['outputs'].message_type = _PRIMITIVETRANSFORM_OUTPUTSENTRY
-_PRIMITIVETRANSFORM.fields_by_name['side_inputs'].message_type = _PRIMITIVETRANSFORM_SIDEINPUTSENTRY
-_FUNCTIONSPEC.fields_by_name['data'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_SIDEINPUT.fields_by_name['input'].message_type = _TARGET
-_SIDEINPUT.fields_by_name['view_fn'].message_type = _FUNCTIONSPEC
-_CODER.fields_by_name['function_spec'].message_type = _FUNCTIONSPEC
-_REMOTEGRPCPORT.fields_by_name['api_service_descriptor'].message_type = _APISERVICEDESCRIPTOR
-_INSTRUCTIONREQUEST.fields_by_name['register'].message_type = _REGISTERREQUEST
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle'].message_type = _PROCESSBUNDLEREQUEST
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_progress'].message_type = _PROCESSBUNDLEPROGRESSREQUEST
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_split'].message_type = _PROCESSBUNDLESPLITREQUEST
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
-  _INSTRUCTIONREQUEST.fields_by_name['register'])
-_INSTRUCTIONREQUEST.fields_by_name['register'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
-  _INSTRUCTIONREQUEST.fields_by_name['process_bundle'])
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
-  _INSTRUCTIONREQUEST.fields_by_name['process_bundle_progress'])
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_progress'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONREQUEST.oneofs_by_name['request'].fields.append(
-  _INSTRUCTIONREQUEST.fields_by_name['process_bundle_split'])
-_INSTRUCTIONREQUEST.fields_by_name['process_bundle_split'].containing_oneof = _INSTRUCTIONREQUEST.oneofs_by_name['request']
-_INSTRUCTIONRESPONSE.fields_by_name['register'].message_type = _REGISTERRESPONSE
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle'].message_type = _PROCESSBUNDLERESPONSE
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_progress'].message_type = _PROCESSBUNDLEPROGRESSRESPONSE
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_split'].message_type = _PROCESSBUNDLESPLITRESPONSE
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
-  _INSTRUCTIONRESPONSE.fields_by_name['register'])
-_INSTRUCTIONRESPONSE.fields_by_name['register'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
-  _INSTRUCTIONRESPONSE.fields_by_name['process_bundle'])
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
-  _INSTRUCTIONRESPONSE.fields_by_name['process_bundle_progress'])
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_progress'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_INSTRUCTIONRESPONSE.oneofs_by_name['response'].fields.append(
-  _INSTRUCTIONRESPONSE.fields_by_name['process_bundle_split'])
-_INSTRUCTIONRESPONSE.fields_by_name['process_bundle_split'].containing_oneof = _INSTRUCTIONRESPONSE.oneofs_by_name['response']
-_REGISTERREQUEST.fields_by_name['process_bundle_descriptor'].message_type = _PROCESSBUNDLEDESCRIPTOR
-_PROCESSBUNDLEDESCRIPTOR.fields_by_name['primitive_transform'].message_type = _PRIMITIVETRANSFORM
-_PROCESSBUNDLEDESCRIPTOR.fields_by_name['coders'].message_type = _CODER
-_PROCESSBUNDLEREQUEST.fields_by_name['cache_tokens'].message_type = _CACHETOKEN
-_PRIMITIVETRANSFORMSPLIT.fields_by_name['completed_restriction'].message_type = _FUNCTIONSPEC
-_PRIMITIVETRANSFORMSPLIT.fields_by_name['remaining_restriction'].message_type = _FUNCTIONSPEC
-_PROCESSBUNDLESPLITRESPONSE.fields_by_name['splits'].message_type = _PRIMITIVETRANSFORMSPLIT
-_ELEMENTS_DATA.fields_by_name['target'].message_type = _TARGET
-_ELEMENTS_DATA.containing_type = _ELEMENTS
-_ELEMENTS.fields_by_name['data'].message_type = _ELEMENTS_DATA
-_STATEREQUEST.fields_by_name['state_key'].message_type = _STATEKEY
-_STATEREQUEST.fields_by_name['get'].message_type = _STATEGETREQUEST
-_STATEREQUEST.fields_by_name['append'].message_type = _STATEAPPENDREQUEST
-_STATEREQUEST.fields_by_name['clear'].message_type = _STATECLEARREQUEST
-_STATEREQUEST.oneofs_by_name['request'].fields.append(
-  _STATEREQUEST.fields_by_name['get'])
-_STATEREQUEST.fields_by_name['get'].containing_oneof = _STATEREQUEST.oneofs_by_name['request']
-_STATEREQUEST.oneofs_by_name['request'].fields.append(
-  _STATEREQUEST.fields_by_name['append'])
-_STATEREQUEST.fields_by_name['append'].containing_oneof = _STATEREQUEST.oneofs_by_name['request']
-_STATEREQUEST.oneofs_by_name['request'].fields.append(
-  _STATEREQUEST.fields_by_name['clear'])
-_STATEREQUEST.fields_by_name['clear'].containing_oneof = _STATEREQUEST.oneofs_by_name['request']
-_STATERESPONSE.fields_by_name['get'].message_type = _STATEGETRESPONSE
-_STATERESPONSE.fields_by_name['append'].message_type = _STATEAPPENDRESPONSE
-_STATERESPONSE.fields_by_name['clear'].message_type = _STATECLEARRESPONSE
-_STATERESPONSE.oneofs_by_name['response'].fields.append(
-  _STATERESPONSE.fields_by_name['get'])
-_STATERESPONSE.fields_by_name['get'].containing_oneof = _STATERESPONSE.oneofs_by_name['response']
-_STATERESPONSE.oneofs_by_name['response'].fields.append(
-  _STATERESPONSE.fields_by_name['append'])
-_STATERESPONSE.fields_by_name['append'].containing_oneof = _STATERESPONSE.oneofs_by_name['response']
-_STATERESPONSE.oneofs_by_name['response'].fields.append(
-  _STATERESPONSE.fields_by_name['clear'])
-_STATERESPONSE.fields_by_name['clear'].containing_oneof = _STATERESPONSE.oneofs_by_name['response']
-_CACHETOKEN.fields_by_name['target'].message_type = _TARGET
-_STATEKEY.fields_by_name['target'].message_type = _TARGET
-_STATEGETRESPONSE.fields_by_name['stream'].message_type = _CONTINUABLESTREAM
-_LOGENTRY_LIST.fields_by_name['log_entries'].message_type = _LOGENTRY
-_LOGENTRY_LIST.containing_type = _LOGENTRY
-_LOGENTRY.fields_by_name['severity'].enum_type = _LOGENTRY_SEVERITY
-_LOGENTRY.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP
-_LOGENTRY_SEVERITY.containing_type = _LOGENTRY
-_APISERVICEDESCRIPTOR.fields_by_name['oauth2_client_credentials_grant'].message_type = _OAUTH2CLIENTCREDENTIALSGRANT
-_APISERVICEDESCRIPTOR.oneofs_by_name['authentication'].fields.append(
-  _APISERVICEDESCRIPTOR.fields_by_name['oauth2_client_credentials_grant'])
-_APISERVICEDESCRIPTOR.fields_by_name['oauth2_client_credentials_grant'].containing_oneof = _APISERVICEDESCRIPTOR.oneofs_by_name['authentication']
-DESCRIPTOR.message_types_by_name['Target'] = _TARGET
-DESCRIPTOR.message_types_by_name['PCollection'] = _PCOLLECTION
-DESCRIPTOR.message_types_by_name['PrimitiveTransform'] = _PRIMITIVETRANSFORM
-DESCRIPTOR.message_types_by_name['FunctionSpec'] = _FUNCTIONSPEC
-DESCRIPTOR.message_types_by_name['SideInput'] = _SIDEINPUT
-DESCRIPTOR.message_types_by_name['Coder'] = _CODER
-DESCRIPTOR.message_types_by_name['RemoteGrpcPort'] = _REMOTEGRPCPORT
-DESCRIPTOR.message_types_by_name['InstructionRequest'] = _INSTRUCTIONREQUEST
-DESCRIPTOR.message_types_by_name['InstructionResponse'] = _INSTRUCTIONRESPONSE
-DESCRIPTOR.message_types_by_name['RegisterRequest'] = _REGISTERREQUEST
-DESCRIPTOR.message_types_by_name['RegisterResponse'] = _REGISTERRESPONSE
-DESCRIPTOR.message_types_by_name['ProcessBundleDescriptor'] = _PROCESSBUNDLEDESCRIPTOR
-DESCRIPTOR.message_types_by_name['ProcessBundleRequest'] = _PROCESSBUNDLEREQUEST
-DESCRIPTOR.message_types_by_name['ProcessBundleResponse'] = _PROCESSBUNDLERESPONSE
-DESCRIPTOR.message_types_by_name['ProcessBundleProgressRequest'] = _PROCESSBUNDLEPROGRESSREQUEST
-DESCRIPTOR.message_types_by_name['ProcessBundleProgressResponse'] = _PROCESSBUNDLEPROGRESSRESPONSE
-DESCRIPTOR.message_types_by_name['ProcessBundleSplitRequest'] = _PROCESSBUNDLESPLITREQUEST
-DESCRIPTOR.message_types_by_name['ElementCountRestriction'] = _ELEMENTCOUNTRESTRICTION
-DESCRIPTOR.message_types_by_name['ElementCountSkipRestriction'] = _ELEMENTCOUNTSKIPRESTRICTION
-DESCRIPTOR.message_types_by_name['PrimitiveTransformSplit'] = _PRIMITIVETRANSFORMSPLIT
-DESCRIPTOR.message_types_by_name['ProcessBundleSplitResponse'] = _PROCESSBUNDLESPLITRESPONSE
-DESCRIPTOR.message_types_by_name['Elements'] = _ELEMENTS
-DESCRIPTOR.message_types_by_name['StateRequest'] = _STATEREQUEST
-DESCRIPTOR.message_types_by_name['StateResponse'] = _STATERESPONSE
-DESCRIPTOR.message_types_by_name['CacheToken'] = _CACHETOKEN
-DESCRIPTOR.message_types_by_name['StateKey'] = _STATEKEY
-DESCRIPTOR.message_types_by_name['ContinuableStream'] = _CONTINUABLESTREAM
-DESCRIPTOR.message_types_by_name['StateGetRequest'] = _STATEGETREQUEST
-DESCRIPTOR.message_types_by_name['StateGetResponse'] = _STATEGETRESPONSE
-DESCRIPTOR.message_types_by_name['StateAppendRequest'] = _STATEAPPENDREQUEST
-DESCRIPTOR.message_types_by_name['StateAppendResponse'] = _STATEAPPENDRESPONSE
-DESCRIPTOR.message_types_by_name['StateClearRequest'] = _STATECLEARREQUEST
-DESCRIPTOR.message_types_by_name['StateClearResponse'] = _STATECLEARRESPONSE
-DESCRIPTOR.message_types_by_name['LogEntry'] = _LOGENTRY
-DESCRIPTOR.message_types_by_name['LogControl'] = _LOGCONTROL
-DESCRIPTOR.message_types_by_name['ApiServiceDescriptor'] = _APISERVICEDESCRIPTOR
-DESCRIPTOR.message_types_by_name['OAuth2ClientCredentialsGrant'] = _OAUTH2CLIENTCREDENTIALSGRANT
-DESCRIPTOR.message_types_by_name['DockerContainer'] = _DOCKERCONTAINER
-
-Target = _reflection.GeneratedProtocolMessageType('Target', (_message.Message,), dict(
-
-  List = _reflection.GeneratedProtocolMessageType('List', (_message.Message,), dict(
-    DESCRIPTOR = _TARGET_LIST,
-    __module__ = 'beam_fn_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Target.List)
-    ))
-  ,
-  DESCRIPTOR = _TARGET,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Target)
-  ))
-_sym_db.RegisterMessage(Target)
-_sym_db.RegisterMessage(Target.List)
-
-PCollection = _reflection.GeneratedProtocolMessageType('PCollection', (_message.Message,), dict(
-  DESCRIPTOR = _PCOLLECTION,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PCollection)
-  ))
-_sym_db.RegisterMessage(PCollection)
-
-PrimitiveTransform = _reflection.GeneratedProtocolMessageType('PrimitiveTransform', (_message.Message,), dict(
-
-  InputsEntry = _reflection.GeneratedProtocolMessageType('InputsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PRIMITIVETRANSFORM_INPUTSENTRY,
-    __module__ = 'beam_fn_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform.InputsEntry)
-    ))
-  ,
-
-  OutputsEntry = _reflection.GeneratedProtocolMessageType('OutputsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PRIMITIVETRANSFORM_OUTPUTSENTRY,
-    __module__ = 'beam_fn_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform.OutputsEntry)
-    ))
-  ,
-
-  SideInputsEntry = _reflection.GeneratedProtocolMessageType('SideInputsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PRIMITIVETRANSFORM_SIDEINPUTSENTRY,
-    __module__ = 'beam_fn_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform.SideInputsEntry)
-    ))
-  ,
-  DESCRIPTOR = _PRIMITIVETRANSFORM,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransform)
-  ))
-_sym_db.RegisterMessage(PrimitiveTransform)
-_sym_db.RegisterMessage(PrimitiveTransform.InputsEntry)
-_sym_db.RegisterMessage(PrimitiveTransform.OutputsEntry)
-_sym_db.RegisterMessage(PrimitiveTransform.SideInputsEntry)
-
-FunctionSpec = _reflection.GeneratedProtocolMessageType('FunctionSpec', (_message.Message,), dict(
-  DESCRIPTOR = _FUNCTIONSPEC,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.FunctionSpec)
-  ))
-_sym_db.RegisterMessage(FunctionSpec)
-
-SideInput = _reflection.GeneratedProtocolMessageType('SideInput', (_message.Message,), dict(
-  DESCRIPTOR = _SIDEINPUT,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.SideInput)
-  ))
-_sym_db.RegisterMessage(SideInput)
-
-Coder = _reflection.GeneratedProtocolMessageType('Coder', (_message.Message,), dict(
-  DESCRIPTOR = _CODER,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Coder)
-  ))
-_sym_db.RegisterMessage(Coder)
-
-RemoteGrpcPort = _reflection.GeneratedProtocolMessageType('RemoteGrpcPort', (_message.Message,), dict(
-  DESCRIPTOR = _REMOTEGRPCPORT,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.RemoteGrpcPort)
-  ))
-_sym_db.RegisterMessage(RemoteGrpcPort)
-
-InstructionRequest = _reflection.GeneratedProtocolMessageType('InstructionRequest', (_message.Message,), dict(
-  DESCRIPTOR = _INSTRUCTIONREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.InstructionRequest)
-  ))
-_sym_db.RegisterMessage(InstructionRequest)
-
-InstructionResponse = _reflection.GeneratedProtocolMessageType('InstructionResponse', (_message.Message,), dict(
-  DESCRIPTOR = _INSTRUCTIONRESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.InstructionResponse)
-  ))
-_sym_db.RegisterMessage(InstructionResponse)
-
-RegisterRequest = _reflection.GeneratedProtocolMessageType('RegisterRequest', (_message.Message,), dict(
-  DESCRIPTOR = _REGISTERREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.RegisterRequest)
-  ))
-_sym_db.RegisterMessage(RegisterRequest)
-
-RegisterResponse = _reflection.GeneratedProtocolMessageType('RegisterResponse', (_message.Message,), dict(
-  DESCRIPTOR = _REGISTERRESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.RegisterResponse)
-  ))
-_sym_db.RegisterMessage(RegisterResponse)
-
-ProcessBundleDescriptor = _reflection.GeneratedProtocolMessageType('ProcessBundleDescriptor', (_message.Message,), dict(
-  DESCRIPTOR = _PROCESSBUNDLEDESCRIPTOR,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleDescriptor)
-  ))
-_sym_db.RegisterMessage(ProcessBundleDescriptor)
-
-ProcessBundleRequest = _reflection.GeneratedProtocolMessageType('ProcessBundleRequest', (_message.Message,), dict(
-  DESCRIPTOR = _PROCESSBUNDLEREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleRequest)
-  ))
-_sym_db.RegisterMessage(ProcessBundleRequest)
-
-ProcessBundleResponse = _reflection.GeneratedProtocolMessageType('ProcessBundleResponse', (_message.Message,), dict(
-  DESCRIPTOR = _PROCESSBUNDLERESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleResponse)
-  ))
-_sym_db.RegisterMessage(ProcessBundleResponse)
-
-ProcessBundleProgressRequest = _reflection.GeneratedProtocolMessageType('ProcessBundleProgressRequest', (_message.Message,), dict(
-  DESCRIPTOR = _PROCESSBUNDLEPROGRESSREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleProgressRequest)
-  ))
-_sym_db.RegisterMessage(ProcessBundleProgressRequest)
-
-ProcessBundleProgressResponse = _reflection.GeneratedProtocolMessageType('ProcessBundleProgressResponse', (_message.Message,), dict(
-  DESCRIPTOR = _PROCESSBUNDLEPROGRESSRESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleProgressResponse)
-  ))
-_sym_db.RegisterMessage(ProcessBundleProgressResponse)
-
-ProcessBundleSplitRequest = _reflection.GeneratedProtocolMessageType('ProcessBundleSplitRequest', (_message.Message,), dict(
-  DESCRIPTOR = _PROCESSBUNDLESPLITREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleSplitRequest)
-  ))
-_sym_db.RegisterMessage(ProcessBundleSplitRequest)
-
-ElementCountRestriction = _reflection.GeneratedProtocolMessageType('ElementCountRestriction', (_message.Message,), dict(
-  DESCRIPTOR = _ELEMENTCOUNTRESTRICTION,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ElementCountRestriction)
-  ))
-_sym_db.RegisterMessage(ElementCountRestriction)
-
-ElementCountSkipRestriction = _reflection.GeneratedProtocolMessageType('ElementCountSkipRestriction', (_message.Message,), dict(
-  DESCRIPTOR = _ELEMENTCOUNTSKIPRESTRICTION,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ElementCountSkipRestriction)
-  ))
-_sym_db.RegisterMessage(ElementCountSkipRestriction)
-
-PrimitiveTransformSplit = _reflection.GeneratedProtocolMessageType('PrimitiveTransformSplit', (_message.Message,), dict(
-  DESCRIPTOR = _PRIMITIVETRANSFORMSPLIT,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.PrimitiveTransformSplit)
-  ))
-_sym_db.RegisterMessage(PrimitiveTransformSplit)
-
-ProcessBundleSplitResponse = _reflection.GeneratedProtocolMessageType('ProcessBundleSplitResponse', (_message.Message,), dict(
-  DESCRIPTOR = _PROCESSBUNDLESPLITRESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ProcessBundleSplitResponse)
-  ))
-_sym_db.RegisterMessage(ProcessBundleSplitResponse)
-
-Elements = _reflection.GeneratedProtocolMessageType('Elements', (_message.Message,), dict(
-
-  Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), dict(
-    DESCRIPTOR = _ELEMENTS_DATA,
-    __module__ = 'beam_fn_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Elements.Data)
-    ))
-  ,
-  DESCRIPTOR = _ELEMENTS,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.Elements)
-  ))
-_sym_db.RegisterMessage(Elements)
-_sym_db.RegisterMessage(Elements.Data)
-
-StateRequest = _reflection.GeneratedProtocolMessageType('StateRequest', (_message.Message,), dict(
-  DESCRIPTOR = _STATEREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateRequest)
-  ))
-_sym_db.RegisterMessage(StateRequest)
-
-StateResponse = _reflection.GeneratedProtocolMessageType('StateResponse', (_message.Message,), dict(
-  DESCRIPTOR = _STATERESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateResponse)
-  ))
-_sym_db.RegisterMessage(StateResponse)
-
-CacheToken = _reflection.GeneratedProtocolMessageType('CacheToken', (_message.Message,), dict(
-  DESCRIPTOR = _CACHETOKEN,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.CacheToken)
-  ))
-_sym_db.RegisterMessage(CacheToken)
-
-StateKey = _reflection.GeneratedProtocolMessageType('StateKey', (_message.Message,), dict(
-  DESCRIPTOR = _STATEKEY,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateKey)
-  ))
-_sym_db.RegisterMessage(StateKey)
-
-ContinuableStream = _reflection.GeneratedProtocolMessageType('ContinuableStream', (_message.Message,), dict(
-  DESCRIPTOR = _CONTINUABLESTREAM,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ContinuableStream)
-  ))
-_sym_db.RegisterMessage(ContinuableStream)
-
-StateGetRequest = _reflection.GeneratedProtocolMessageType('StateGetRequest', (_message.Message,), dict(
-  DESCRIPTOR = _STATEGETREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateGetRequest)
-  ))
-_sym_db.RegisterMessage(StateGetRequest)
-
-StateGetResponse = _reflection.GeneratedProtocolMessageType('StateGetResponse', (_message.Message,), dict(
-  DESCRIPTOR = _STATEGETRESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateGetResponse)
-  ))
-_sym_db.RegisterMessage(StateGetResponse)
-
-StateAppendRequest = _reflection.GeneratedProtocolMessageType('StateAppendRequest', (_message.Message,), dict(
-  DESCRIPTOR = _STATEAPPENDREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateAppendRequest)
-  ))
-_sym_db.RegisterMessage(StateAppendRequest)
-
-StateAppendResponse = _reflection.GeneratedProtocolMessageType('StateAppendResponse', (_message.Message,), dict(
-  DESCRIPTOR = _STATEAPPENDRESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateAppendResponse)
-  ))
-_sym_db.RegisterMessage(StateAppendResponse)
-
-StateClearRequest = _reflection.GeneratedProtocolMessageType('StateClearRequest', (_message.Message,), dict(
-  DESCRIPTOR = _STATECLEARREQUEST,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateClearRequest)
-  ))
-_sym_db.RegisterMessage(StateClearRequest)
-
-StateClearResponse = _reflection.GeneratedProtocolMessageType('StateClearResponse', (_message.Message,), dict(
-  DESCRIPTOR = _STATECLEARRESPONSE,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.StateClearResponse)
-  ))
-_sym_db.RegisterMessage(StateClearResponse)
-
-LogEntry = _reflection.GeneratedProtocolMessageType('LogEntry', (_message.Message,), dict(
-
-  List = _reflection.GeneratedProtocolMessageType('List', (_message.Message,), dict(
-    DESCRIPTOR = _LOGENTRY_LIST,
-    __module__ = 'beam_fn_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.LogEntry.List)
-    ))
-  ,
-  DESCRIPTOR = _LOGENTRY,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.LogEntry)
-  ))
-_sym_db.RegisterMessage(LogEntry)
-_sym_db.RegisterMessage(LogEntry.List)
-
-LogControl = _reflection.GeneratedProtocolMessageType('LogControl', (_message.Message,), dict(
-  DESCRIPTOR = _LOGCONTROL,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.LogControl)
-  ))
-_sym_db.RegisterMessage(LogControl)
-
-ApiServiceDescriptor = _reflection.GeneratedProtocolMessageType('ApiServiceDescriptor', (_message.Message,), dict(
-  DESCRIPTOR = _APISERVICEDESCRIPTOR,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.ApiServiceDescriptor)
-  ))
-_sym_db.RegisterMessage(ApiServiceDescriptor)
-
-OAuth2ClientCredentialsGrant = _reflection.GeneratedProtocolMessageType('OAuth2ClientCredentialsGrant', (_message.Message,), dict(
-  DESCRIPTOR = _OAUTH2CLIENTCREDENTIALSGRANT,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.OAuth2ClientCredentialsGrant)
-  ))
-_sym_db.RegisterMessage(OAuth2ClientCredentialsGrant)
-
-DockerContainer = _reflection.GeneratedProtocolMessageType('DockerContainer', (_message.Message,), dict(
-  DESCRIPTOR = _DOCKERCONTAINER,
-  __module__ = 'beam_fn_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.fn.v1.DockerContainer)
-  ))
-_sym_db.RegisterMessage(DockerContainer)
-
-
-DESCRIPTOR.has_options = True
-DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\025org.apache.beam.fn.v1B\tBeamFnApi'))
-_PRIMITIVETRANSFORM_INPUTSENTRY.has_options = True
-_PRIMITIVETRANSFORM_INPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PRIMITIVETRANSFORM_OUTPUTSENTRY.has_options = True
-_PRIMITIVETRANSFORM_OUTPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY.has_options = True
-_PRIMITIVETRANSFORM_SIDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-try:
-  # THESE ELEMENTS WILL BE DEPRECATED.
-  # Please use the generated *_pb2_grpc.py files instead.
-  import grpc
-  from grpc.framework.common import cardinality
-  from grpc.framework.interfaces.face import utilities as face_utilities
-  from grpc.beta import implementations as beta_implementations
-  from grpc.beta import interfaces as beta_interfaces
-
-
-  class BeamFnControlStub(object):
-    """
-    Control Plane API
-
-    Progress reporting and splitting still need further vetting. Also, this may change
-    with the addition of new types of instructions/responses related to metrics.
-
-    An API that describes the work that a SDK harness is meant to do.
-    Stable
-    """
-
-    def __init__(self, channel):
-      """Constructor.
-
-      Args:
-        channel: A grpc.Channel.
-      """
-      self.Control = channel.stream_stream(
-          '/org.apache.beam.fn.v1.BeamFnControl/Control',
-          request_serializer=InstructionResponse.SerializeToString,
-          response_deserializer=InstructionRequest.FromString,
-          )
-
-
-  class BeamFnControlServicer(object):
-    """
-    Control Plane API
-
-    Progress reporting and splitting still need further vetting. Also, this may change
-    with the addition of new types of instructions/responses related to metrics.
-
-    An API that describes the work that a SDK harness is meant to do.
-    Stable
-    """
-
-    def Control(self, request_iterator, context):
-      """Instructions sent by the runner to the SDK requesting different types
-      of work.
-      """
-      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-      context.set_details('Method not implemented!')
-      raise NotImplementedError('Method not implemented!')
-
-
-  def add_BeamFnControlServicer_to_server(servicer, server):
-    rpc_method_handlers = {
-        'Control': grpc.stream_stream_rpc_method_handler(
-            servicer.Control,
-            request_deserializer=InstructionResponse.FromString,
-            response_serializer=InstructionRequest.SerializeToString,
-        ),
-    }
-    generic_handler = grpc.method_handlers_generic_handler(
-        'org.apache.beam.fn.v1.BeamFnControl', rpc_method_handlers)
-    server.add_generic_rpc_handlers((generic_handler,))
-
-
-  class BeamFnDataStub(object):
-    """Stable
-    """
-
-    def __init__(self, channel):
-      """Constructor.
-
-      Args:
-        channel: A grpc.Channel.
-      """
-      self.Data = channel.stream_stream(
-          '/org.apache.beam.fn.v1.BeamFnData/Data',
-          request_serializer=Elements.SerializeToString,
-          response_deserializer=Elements.FromString,
-          )
-
-
-  class BeamFnDataServicer(object):
-    """Stable
-    """
-
-    def Data(self, request_iterator, context):
-      """Used to send data between harnesses.
-      """
-      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-      context.set_details('Method not implemented!')
-      raise NotImplementedError('Method not implemented!')
-
-
-  def add_BeamFnDataServicer_to_server(servicer, server):
-    rpc_method_handlers = {
-        'Data': grpc.stream_stream_rpc_method_handler(
-            servicer.Data,
-            request_deserializer=Elements.FromString,
-            response_serializer=Elements.SerializeToString,
-        ),
-    }
-    generic_handler = grpc.method_handlers_generic_handler(
-        'org.apache.beam.fn.v1.BeamFnData', rpc_method_handlers)
-    server.add_generic_rpc_handlers((generic_handler,))
-
-
-  class BeamFnStateStub(object):
-
-    def __init__(self, channel):
-      """Constructor.
-
-      Args:
-        channel: A grpc.Channel.
-      """
-      self.State = channel.stream_stream(
-          '/org.apache.beam.fn.v1.BeamFnState/State',
-          request_serializer=StateRequest.SerializeToString,
-          response_deserializer=StateResponse.FromString,
-          )
-
-
-  class BeamFnStateServicer(object):
-
-    def State(self, request_iterator, context):
-      """Used to get/append/clear state stored by the runner on behalf of the SDK.
-      """
-      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-      context.set_details('Method not implemented!')
-      raise NotImplementedError('Method not implemented!')
-
-
-  def add_BeamFnStateServicer_to_server(servicer, server):
-    rpc_method_handlers = {
-        'State': grpc.stream_stream_rpc_method_handler(
-            servicer.State,
-            request_deserializer=StateRequest.FromString,
-            response_serializer=StateResponse.SerializeToString,
-        ),
-    }
-    generic_handler = grpc.method_handlers_generic_handler(
-        'org.apache.beam.fn.v1.BeamFnState', rpc_method_handlers)
-    server.add_generic_rpc_handlers((generic_handler,))
-
-
-  class BeamFnLoggingStub(object):
-    """Stable
-    """
-
-    def __init__(self, channel):
-      """Constructor.
-
-      Args:
-        channel: A grpc.Channel.
-      """
-      self.Logging = channel.stream_stream(
-          '/org.apache.beam.fn.v1.BeamFnLogging/Logging',
-          request_serializer=LogEntry.List.SerializeToString,
-          response_deserializer=LogControl.FromString,
-          )
-
-
-  class BeamFnLoggingServicer(object):
-    """Stable
-    """
-
-    def Logging(self, request_iterator, context):
-      """Allows for the SDK to emit log entries which the runner can
-      associate with the active job.
-      """
-      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-      context.set_details('Method not implemented!')
-      raise NotImplementedError('Method not implemented!')
-
-
-  def add_BeamFnLoggingServicer_to_server(servicer, server):
-    rpc_method_handlers = {
-        'Logging': grpc.stream_stream_rpc_method_handler(
-            servicer.Logging,
-            request_deserializer=LogEntry.List.FromString,
-            response_serializer=LogControl.SerializeToString,
-        ),
-    }
-    generic_handler = grpc.method_handlers_generic_handler(
-        'org.apache.beam.fn.v1.BeamFnLogging', rpc_method_handlers)
-    server.add_generic_rpc_handlers((generic_handler,))
-
-
-  class BetaBeamFnControlServicer(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    """
-    Control Plane API
-
-    Progress reporting and splitting still need further vetting. Also, this may change
-    with the addition of new types of instructions/responses related to metrics.
-
-    An API that describes the work that a SDK harness is meant to do.
-    Stable
-    """
-    def Control(self, request_iterator, context):
-      """Instructions sent by the runner to the SDK requesting different types
-      of work.
-      """
-      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
-  class BetaBeamFnControlStub(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    """
-    Control Plane API
-
-    Progress reporting and splitting still need further vetting. Also, this may change
-    with the addition of new types of instructions/responses related to metrics.
-
-    An API that describes the work that a SDK harness is meant to do.
-    Stable
-    """
-    def Control(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-      """Instructions sent by the runner to the SDK requesting different types
-      of work.
-      """
-      raise NotImplementedError()
-
-
-  def beta_create_BeamFnControl_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionResponse.FromString,
-    }
-    response_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionRequest.SerializeToString,
-    }
-    method_implementations = {
-      ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): face_utilities.stream_stream_inline(servicer.Control),
-    }
-    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-    return beta_implementations.server(method_implementations, options=server_options)
-
-
-  def beta_create_BeamFnControl_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionResponse.SerializeToString,
-    }
-    response_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnControl', 'Control'): InstructionRequest.FromString,
-    }
-    cardinalities = {
-      'Control': cardinality.Cardinality.STREAM_STREAM,
-    }
-    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-    return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnControl', cardinalities, options=stub_options)
-
-
-  class BetaBeamFnDataServicer(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    """Stable
-    """
-    def Data(self, request_iterator, context):
-      """Used to send data between harnesses.
-      """
-      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
-  class BetaBeamFnDataStub(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    """Stable
-    """
-    def Data(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-      """Used to send data between harnesses.
-      """
-      raise NotImplementedError()
-
-
-  def beta_create_BeamFnData_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.FromString,
-    }
-    response_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.SerializeToString,
-    }
-    method_implementations = {
-      ('org.apache.beam.fn.v1.BeamFnData', 'Data'): face_utilities.stream_stream_inline(servicer.Data),
-    }
-    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-    return beta_implementations.server(method_implementations, options=server_options)
-
-
-  def beta_create_BeamFnData_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.SerializeToString,
-    }
-    response_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnData', 'Data'): Elements.FromString,
-    }
-    cardinalities = {
-      'Data': cardinality.Cardinality.STREAM_STREAM,
-    }
-    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-    return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnData', cardinalities, options=stub_options)
-
-
-  class BetaBeamFnStateServicer(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    def State(self, request_iterator, context):
-      """Used to get/append/clear state stored by the runner on behalf of the SDK.
-      """
-      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
-  class BetaBeamFnStateStub(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    def State(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-      """Used to get/append/clear state stored by the runner on behalf of the SDK.
-      """
-      raise NotImplementedError()
-
-
-  def beta_create_BeamFnState_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateRequest.FromString,
-    }
-    response_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateResponse.SerializeToString,
-    }
-    method_implementations = {
-      ('org.apache.beam.fn.v1.BeamFnState', 'State'): face_utilities.stream_stream_inline(servicer.State),
-    }
-    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-    return beta_implementations.server(method_implementations, options=server_options)
-
-
-  def beta_create_BeamFnState_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateRequest.SerializeToString,
-    }
-    response_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnState', 'State'): StateResponse.FromString,
-    }
-    cardinalities = {
-      'State': cardinality.Cardinality.STREAM_STREAM,
-    }
-    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-    return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnState', cardinalities, options=stub_options)
-
-
-  class BetaBeamFnLoggingServicer(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    """Stable
-    """
-    def Logging(self, request_iterator, context):
-      """Allows for the SDK to emit log entries which the runner can
-      associate with the active job.
-      """
-      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
-  class BetaBeamFnLoggingStub(object):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This class was generated
-    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
-    """Stable
-    """
-    def Logging(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-      """Allows for the SDK to emit log entries which the runner can
-      associate with the active job.
-      """
-      raise NotImplementedError()
-
-
-  def beta_create_BeamFnLogging_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogEntry.List.FromString,
-    }
-    response_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogControl.SerializeToString,
-    }
-    method_implementations = {
-      ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): face_utilities.stream_stream_inline(servicer.Logging),
-    }
-    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-    return beta_implementations.server(method_implementations, options=server_options)
-
-
-  def beta_create_BeamFnLogging_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-    """The Beta API is deprecated for 0.15.0 and later.
-
-    It is recommended to use the GA API (classes and functions in this
-    file not marked beta) for all further purposes. This function was
-    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
-    request_serializers = {
-      ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogEntry.List.SerializeToString,
-    }
-    response_deserializers = {
-      ('org.apache.beam.fn.v1.BeamFnLogging', 'Logging'): LogControl.FromString,
-    }
-    cardinalities = {
-      'Logging': cardinality.Cardinality.STREAM_STREAM,
-    }
-    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-    return beta_implementations.dynamic_stub(channel, 'org.apache.beam.fn.v1.BeamFnLogging', cardinalities, options=stub_options)
-except ImportError:
-  pass
-# @@protoc_insertion_point(module_scope)
diff --git a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py b/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
deleted file mode 100644
index 08d7dada14a6..000000000000
--- a/sdks/python/apache_beam/runners/api/beam_fn_api_pb2_grpc.py
+++ /dev/null
@@ -1,205 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-import grpc
-from grpc.framework.common import cardinality
-from grpc.framework.interfaces.face import utilities as face_utilities
-
-import beam_fn_api_pb2 as beam__fn__api__pb2
-
-# This module is experimental. No backwards-compatibility guarantees.
-
-
-class BeamFnControlStub(object):
-  """
-  Control Plane API
-
-  Progress reporting and splitting still need further vetting. Also, this may change
-  with the addition of new types of instructions/responses related to metrics.
-
-  An API that describes the work that a SDK harness is meant to do.
-  Stable
-  """
-
-  def __init__(self, channel):
-    """Constructor.
-
-    Args:
-      channel: A grpc.Channel.
-    """
-    self.Control = channel.stream_stream(
-        '/org.apache.beam.fn.v1.BeamFnControl/Control',
-        request_serializer=beam__fn__api__pb2.InstructionResponse.SerializeToString,
-        response_deserializer=beam__fn__api__pb2.InstructionRequest.FromString,
-        )
-
-
-class BeamFnControlServicer(object):
-  """
-  Control Plane API
-
-  Progress reporting and splitting still need further vetting. Also, this may change
-  with the addition of new types of instructions/responses related to metrics.
-
-  An API that describes the work that a SDK harness is meant to do.
-  Stable
-  """
-
-  def Control(self, request_iterator, context):
-    """Instructions sent by the runner to the SDK requesting different types
-    of work.
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnControlServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'Control': grpc.stream_stream_rpc_method_handler(
-          servicer.Control,
-          request_deserializer=beam__fn__api__pb2.InstructionResponse.FromString,
-          response_serializer=beam__fn__api__pb2.InstructionRequest.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'org.apache.beam.fn.v1.BeamFnControl', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BeamFnDataStub(object):
-  """Stable
-  """
-
-  def __init__(self, channel):
-    """Constructor.
-
-    Args:
-      channel: A grpc.Channel.
-    """
-    self.Data = channel.stream_stream(
-        '/org.apache.beam.fn.v1.BeamFnData/Data',
-        request_serializer=beam__fn__api__pb2.Elements.SerializeToString,
-        response_deserializer=beam__fn__api__pb2.Elements.FromString,
-        )
-
-
-class BeamFnDataServicer(object):
-  """Stable
-  """
-
-  def Data(self, request_iterator, context):
-    """Used to send data between harnesses.
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnDataServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'Data': grpc.stream_stream_rpc_method_handler(
-          servicer.Data,
-          request_deserializer=beam__fn__api__pb2.Elements.FromString,
-          response_serializer=beam__fn__api__pb2.Elements.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'org.apache.beam.fn.v1.BeamFnData', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BeamFnStateStub(object):
-
-  def __init__(self, channel):
-    """Constructor.
-
-    Args:
-      channel: A grpc.Channel.
-    """
-    self.State = channel.stream_stream(
-        '/org.apache.beam.fn.v1.BeamFnState/State',
-        request_serializer=beam__fn__api__pb2.StateRequest.SerializeToString,
-        response_deserializer=beam__fn__api__pb2.StateResponse.FromString,
-        )
-
-
-class BeamFnStateServicer(object):
-
-  def State(self, request_iterator, context):
-    """Used to get/append/clear state stored by the runner on behalf of the SDK.
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnStateServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'State': grpc.stream_stream_rpc_method_handler(
-          servicer.State,
-          request_deserializer=beam__fn__api__pb2.StateRequest.FromString,
-          response_serializer=beam__fn__api__pb2.StateResponse.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'org.apache.beam.fn.v1.BeamFnState', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BeamFnLoggingStub(object):
-  """Stable
-  """
-
-  def __init__(self, channel):
-    """Constructor.
-
-    Args:
-      channel: A grpc.Channel.
-    """
-    self.Logging = channel.stream_stream(
-        '/org.apache.beam.fn.v1.BeamFnLogging/Logging',
-        request_serializer=beam__fn__api__pb2.LogEntry.List.SerializeToString,
-        response_deserializer=beam__fn__api__pb2.LogControl.FromString,
-        )
-
-
-class BeamFnLoggingServicer(object):
-  """Stable
-  """
-
-  def Logging(self, request_iterator, context):
-    """Allows for the SDK to emit log entries which the runner can
-    associate with the active job.
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_BeamFnLoggingServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'Logging': grpc.stream_stream_rpc_method_handler(
-          servicer.Logging,
-          request_deserializer=beam__fn__api__pb2.LogEntry.List.FromString,
-          response_serializer=beam__fn__api__pb2.LogControl.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'org.apache.beam.fn.v1.BeamFnLogging', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
diff --git a/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py b/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
deleted file mode 100644
index e8793b63da51..000000000000
--- a/sdks/python/apache_beam/runners/api/beam_runner_api_pb2.py
+++ /dev/null
@@ -1,2872 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: beam_runner_api.proto
-
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
-from google.protobuf.internal import enum_type_wrapper
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
-
-# This module is experimental. No backwards-compatibility guarantees.
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
-  name='beam_runner_api.proto',
-  package='org.apache.beam.runner_api.v1',
-  syntax='proto3',
-  serialized_pb=_b('\n\x15\x62\x65\x61m_runner_api.proto\x12\x1dorg.apache.beam.runner_api.v1\x1a\x19google/protobuf/any.proto\"\x8d\x07\n\nComponents\x12M\n\ntransforms\x18\x01 \x03(\x0b\x32\x39.org.apache.beam.runner_api.v1.Components.TransformsEntry\x12Q\n\x0cpcollections\x18\x02 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.Components.PcollectionsEntry\x12`\n\x14windowing_strategies\x18\x03 \x03(\x0b\x32\x42.org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry\x12\x45\n\x06\x63oders\x18\x04 \x03(\x0b\x32\x35.org.apache.beam.runner_api.v1.Components.CodersEntry\x12Q\n\x0c\x65nvironments\x18\x05 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.Components.EnvironmentsEntry\x1a\\\n\x0fTransformsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32).org.apache.beam.runner_api.v1.PTransform:\x02\x38\x01\x1a_\n\x11PcollectionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.PCollection:\x02\x38\x01\x1al\n\x18WindowingStrategiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12?\n\x05value\x18\x02 \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.WindowingStrategy:\x02\x38\x01\x1aS\n\x0b\x43odersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.org.apache.beam.runner_api.v1.Coder:\x02\x38\x01\x1a_\n\x11\x45nvironmentsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.Environment:\x02\x38\x01\"\xe2\x06\n\x15MessageWithComponents\x12=\n\ncomponents\x18\x01 \x01(\x0b\x32).org.apache.beam.runner_api.v1.Components\x12\x35\n\x05\x63oder\x18\x02 \x01(\x0b\x32$.org.apache.beam.runner_api.v1.CoderH\x00\x12H\n\x0f\x63ombine_payload\x18\x03 \x01(\x0b\x32-.org.apache.beam.runner_api.v1.CombinePayloadH\x00\x12K\n\x11sdk_function_spec\x18\x04 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpecH\x00\x12\x45\n\x0epar_do_payload\x18\x06 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.ParDoPayloadH\x00\x12?\n\nptransform\x18\x07 \x01(\x0b\x32).org.apache.beam.runner_api.v1.PTransformH\x00\x12\x41\n\x0bpcollection\x18\x08 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.PCollectionH\x00\x12\x42\n\x0cread_payload\x18\t \x01(\x0b\x32*.org.apache.beam.runner_api.v1.ReadPayloadH\x00\x12>\n\nside_input\x18\x0b \x01(\x0b\x32(.org.apache.beam.runner_api.v1.SideInputH\x00\x12O\n\x13window_into_payload\x18\x0c \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.WindowIntoPayloadH\x00\x12N\n\x12windowing_strategy\x18\r \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.WindowingStrategyH\x00\x12\x44\n\rfunction_spec\x18\x0e \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpecH\x00\x42\x06\n\x04root\"\xa7\x01\n\x08Pipeline\x12=\n\ncomponents\x18\x01 \x01(\x0b\x32).org.apache.beam.runner_api.v1.Components\x12\x1a\n\x12root_transform_ids\x18\x02 \x03(\t\x12@\n\x0c\x64isplay_data\x18\x03 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.DisplayData\"\xa4\x03\n\nPTransform\x12\x13\n\x0bunique_name\x18\x05 \x01(\t\x12\x39\n\x04spec\x18\x01 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpec\x12\x15\n\rsubtransforms\x18\x02 \x03(\t\x12\x45\n\x06inputs\x18\x03 \x03(\x0b\x32\x35.org.apache.beam.runner_api.v1.PTransform.InputsEntry\x12G\n\x07outputs\x18\x04 \x03(\x0b\x32\x36.org.apache.beam.runner_api.v1.PTransform.OutputsEntry\x12@\n\x0c\x64isplay_data\x18\x06 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.DisplayData\x1a-\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a.\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd3\x01\n\x0bPCollection\x12\x13\n\x0bunique_name\x18\x01 \x01(\t\x12\x10\n\x08\x63oder_id\x18\x02 \x01(\t\x12<\n\nis_bounded\x18\x03 \x01(\x0e\x32(.org.apache.beam.runner_api.v1.IsBounded\x12\x1d\n\x15windowing_strategy_id\x18\x04 \x01(\t\x12@\n\x0c\x64isplay_data\x18\x05 \x01(\x0b\x32*.org.apache.beam.runner_api.v1.DisplayData\"\x98\x05\n\x0cParDoPayload\x12=\n\x05\x64o_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12<\n\nparameters\x18\x02 \x03(\x0b\x32(.org.apache.beam.runner_api.v1.Parameter\x12P\n\x0bside_inputs\x18\x03 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry\x12P\n\x0bstate_specs\x18\x04 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry\x12P\n\x0btimer_specs\x18\x05 \x03(\x0b\x32;.org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry\x1a[\n\x0fSideInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.SideInput:\x02\x38\x01\x1a[\n\x0fStateSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.StateSpec:\x02\x38\x01\x1a[\n\x0fTimerSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.TimerSpec:\x02\x38\x01\"\x8b\x01\n\tParameter\x12;\n\x04type\x18\x01 \x01(\x0e\x32-.org.apache.beam.runner_api.v1.Parameter.Type\"A\n\x04Type\x12\n\n\x06WINDOW\x10\x00\x12\x14\n\x10PIPELINE_OPTIONS\x10\x01\x12\x17\n\x13RESTRICTION_TRACKER\x10\x02\"\x0b\n\tStateSpec\"\x0b\n\tTimerSpec\"\x8b\x01\n\x0bReadPayload\x12>\n\x06source\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12<\n\nis_bounded\x18\x02 \x01(\x0e\x32(.org.apache.beam.runner_api.v1.IsBounded\"V\n\x11WindowIntoPayload\x12\x41\n\twindow_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\"\xe1\x02\n\x0e\x43ombinePayload\x12\x42\n\ncombine_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12\x1c\n\x14\x61\x63\x63umulator_coder_id\x18\x02 \x01(\t\x12<\n\nparameters\x18\x03 \x03(\x0b\x32(.org.apache.beam.runner_api.v1.Parameter\x12R\n\x0bside_inputs\x18\x04 \x03(\x0b\x32=.org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry\x1a[\n\x0fSideInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.org.apache.beam.runner_api.v1.SideInput:\x02\x38\x01\"b\n\x05\x43oder\x12<\n\x04spec\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12\x1b\n\x13\x63omponent_coder_ids\x18\x02 \x03(\t\"\xda\x03\n\x11WindowingStrategy\x12\x41\n\twindow_fn\x18\x01 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12@\n\x0cmerge_status\x18\x02 \x01(\x0e\x32*.org.apache.beam.runner_api.v1.MergeStatus\x12\x17\n\x0fwindow_coder_id\x18\x03 \x01(\t\x12\x37\n\x07trigger\x18\x04 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x12J\n\x11\x61\x63\x63umulation_mode\x18\x05 \x01(\x0e\x32/.org.apache.beam.runner_api.v1.AccumulationMode\x12>\n\x0boutput_time\x18\x06 \x01(\x0e\x32).org.apache.beam.runner_api.v1.OutputTime\x12H\n\x10\x63losing_behavior\x18\x07 \x01(\x0e\x32..org.apache.beam.runner_api.v1.ClosingBehavior\x12\x18\n\x10\x61llowed_lateness\x18\x08 \x01(\x03\"\xad\r\n\x07Trigger\x12\x44\n\tafter_all\x18\x01 \x01(\x0b\x32/.org.apache.beam.runner_api.v1.Trigger.AfterAllH\x00\x12\x44\n\tafter_any\x18\x02 \x01(\x0b\x32/.org.apache.beam.runner_api.v1.Trigger.AfterAnyH\x00\x12\x46\n\nafter_each\x18\x03 \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.Trigger.AfterEachH\x00\x12V\n\x13\x61\x66ter_end_of_window\x18\x04 \x01(\x0b\x32\x37.org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindowH\x00\x12[\n\x15\x61\x66ter_processing_time\x18\x05 \x01(\x0b\x32:.org.apache.beam.runner_api.v1.Trigger.AfterProcessingTimeH\x00\x12t\n\"after_synchronized_processing_time\x18\x06 \x01(\x0b\x32\x46.org.apache.beam.runner_api.v1.Trigger.AfterSynchronizedProcessingTimeH\x00\x12?\n\x06\x61lways\x18\x0c \x01(\x0b\x32-.org.apache.beam.runner_api.v1.Trigger.AlwaysH\x00\x12\x41\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\x0b\x32..org.apache.beam.runner_api.v1.Trigger.DefaultH\x00\x12L\n\relement_count\x18\x08 \x01(\x0b\x32\x33.org.apache.beam.runner_api.v1.Trigger.ElementCountH\x00\x12=\n\x05never\x18\t \x01(\x0b\x32,.org.apache.beam.runner_api.v1.Trigger.NeverH\x00\x12\x46\n\nor_finally\x18\n \x01(\x0b\x32\x30.org.apache.beam.runner_api.v1.Trigger.OrFinallyH\x00\x12?\n\x06repeat\x18\x0b \x01(\x0b\x32-.org.apache.beam.runner_api.v1.Trigger.RepeatH\x00\x1aG\n\x08\x41\x66terAll\x12;\n\x0bsubtriggers\x18\x01 \x03(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1aG\n\x08\x41\x66terAny\x12;\n\x0bsubtriggers\x18\x01 \x03(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1aH\n\tAfterEach\x12;\n\x0bsubtriggers\x18\x01 \x03(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1a\x8f\x01\n\x10\x41\x66terEndOfWindow\x12=\n\rearly_firings\x18\x01 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x12<\n\x0clate_firings\x18\x02 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1a\x66\n\x13\x41\x66terProcessingTime\x12O\n\x14timestamp_transforms\x18\x01 \x03(\x0b\x32\x31.org.apache.beam.runner_api.v1.TimestampTransform\x1a!\n\x1f\x41\x66terSynchronizedProcessingTime\x1a\t\n\x07\x44\x65\x66\x61ult\x1a%\n\x0c\x45lementCount\x12\x15\n\relement_count\x18\x01 \x01(\x05\x1a\x07\n\x05Never\x1a\x08\n\x06\x41lways\x1az\n\tOrFinally\x12\x34\n\x04main\x18\x01 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x12\x37\n\x07\x66inally\x18\x02 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.Trigger\x1a\x44\n\x06Repeat\x12:\n\nsubtrigger\x18\x01 \x01(\x0b\x32&.org.apache.beam.runner_api.v1.TriggerB\t\n\x07trigger\"\x8e\x02\n\x12TimestampTransform\x12H\n\x05\x64\x65lay\x18\x01 \x01(\x0b\x32\x37.org.apache.beam.runner_api.v1.TimestampTransform.DelayH\x00\x12M\n\x08\x61lign_to\x18\x02 \x01(\x0b\x32\x39.org.apache.beam.runner_api.v1.TimestampTransform.AlignToH\x00\x1a\x1d\n\x05\x44\x65lay\x12\x14\n\x0c\x64\x65lay_millis\x18\x01 \x01(\x03\x1a)\n\x07\x41lignTo\x12\x0e\n\x06period\x18\x03 \x01(\x03\x12\x0e\n\x06offset\x18\x04 \x01(\x03\x42\x15\n\x13timestamp_transform\"\xdc\x01\n\tSideInput\x12\x43\n\x0e\x61\x63\x63\x65ss_pattern\x18\x01 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpec\x12?\n\x07view_fn\x18\x02 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\x12I\n\x11window_mapping_fn\x18\x03 \x01(\x0b\x32..org.apache.beam.runner_api.v1.SdkFunctionSpec\"\x1a\n\x0b\x45nvironment\x12\x0b\n\x03url\x18\x01 \x01(\t\"d\n\x0fSdkFunctionSpec\x12\x39\n\x04spec\x18\x01 \x01(\x0b\x32+.org.apache.beam.runner_api.v1.FunctionSpec\x12\x16\n\x0e\x65nvironment_id\x18\x02 \x01(\t\"D\n\x0c\x46unctionSpec\x12\x0b\n\x03urn\x18\x01 \x01(\t\x12\'\n\tparameter\x18\x02 \x01(\x0b\x32\x14.google.protobuf.Any\"\xf7\x03\n\x0b\x44isplayData\x12>\n\x05items\x18\x01 \x03(\x0b\x32/.org.apache.beam.runner_api.v1.DisplayData.Item\x1a\x46\n\nIdentifier\x12\x14\n\x0ctransform_id\x18\x01 \x01(\t\x12\x15\n\rtransform_urn\x18\x02 \x01(\t\x12\x0b\n\x03key\x18\x03 \x01(\t\x1a\xf9\x01\n\x04Item\x12\x41\n\x02id\x18\x01 \x01(\x0b\x32\x35.org.apache.beam.runner_api.v1.DisplayData.Identifier\x12=\n\x04type\x18\x02 \x01(\x0e\x32/.org.apache.beam.runner_api.v1.DisplayData.Type\x12#\n\x05value\x18\x03 \x01(\x0b\x32\x14.google.protobuf.Any\x12)\n\x0bshort_value\x18\x04 \x01(\x0b\x32\x14.google.protobuf.Any\x12\r\n\x05label\x18\x05 \x01(\t\x12\x10\n\x08link_url\x18\x06 \x01(\t\"d\n\x04Type\x12\n\n\x06STRING\x10\x00\x12\x0b\n\x07INTEGER\x10\x01\x12\t\n\x05\x46LOAT\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\r\n\tTIMESTAMP\x10\x04\x12\x0c\n\x08\x44URATION\x10\x05\x12\x0e\n\nJAVA_CLASS\x10\x06*\'\n\tIsBounded\x12\x0b\n\x07\x42OUNDED\x10\x00\x12\r\n\tUNBOUNDED\x10\x01*C\n\x0bMergeStatus\x12\x0f\n\x0bNON_MERGING\x10\x00\x12\x0f\n\x0bNEEDS_MERGE\x10\x01\x12\x12\n\x0e\x41LREADY_MERGED\x10\x02*4\n\x10\x41\x63\x63umulationMode\x12\x0e\n\nDISCARDING\x10\x00\x12\x10\n\x0c\x41\x43\x43UMULATING\x10\x01*8\n\x0f\x43losingBehavior\x12\x0f\n\x0b\x45MIT_ALWAYS\x10\x00\x12\x14\n\x10\x45MIT_IF_NONEMPTY\x10\x01*I\n\nOutputTime\x12\x11\n\rEND_OF_WINDOW\x10\x00\x12\x12\n\x0eLATEST_IN_PANE\x10\x01\x12\x14\n\x10\x45\x41RLIEST_IN_PANE\x10\x02*S\n\nTimeDomain\x12\x0e\n\nEVENT_TIME\x10\x00\x12\x13\n\x0fPROCESSING_TIME\x10\x01\x12 \n\x1cSYNCHRONIZED_PROCESSING_TIME\x10\x02\x42\x31\n$org.apache.beam.sdk.common.runner.v1B\tRunnerApib\x06proto3')
-  ,
-  dependencies=[google_dot_protobuf_dot_any__pb2.DESCRIPTOR,])
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-_ISBOUNDED = _descriptor.EnumDescriptor(
-  name='IsBounded',
-  full_name='org.apache.beam.runner_api.v1.IsBounded',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='BOUNDED', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='UNBOUNDED', index=1, number=1,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=7583,
-  serialized_end=7622,
-)
-_sym_db.RegisterEnumDescriptor(_ISBOUNDED)
-
-IsBounded = enum_type_wrapper.EnumTypeWrapper(_ISBOUNDED)
-_MERGESTATUS = _descriptor.EnumDescriptor(
-  name='MergeStatus',
-  full_name='org.apache.beam.runner_api.v1.MergeStatus',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='NON_MERGING', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='NEEDS_MERGE', index=1, number=1,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='ALREADY_MERGED', index=2, number=2,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=7624,
-  serialized_end=7691,
-)
-_sym_db.RegisterEnumDescriptor(_MERGESTATUS)
-
-MergeStatus = enum_type_wrapper.EnumTypeWrapper(_MERGESTATUS)
-_ACCUMULATIONMODE = _descriptor.EnumDescriptor(
-  name='AccumulationMode',
-  full_name='org.apache.beam.runner_api.v1.AccumulationMode',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='DISCARDING', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='ACCUMULATING', index=1, number=1,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=7693,
-  serialized_end=7745,
-)
-_sym_db.RegisterEnumDescriptor(_ACCUMULATIONMODE)
-
-AccumulationMode = enum_type_wrapper.EnumTypeWrapper(_ACCUMULATIONMODE)
-_CLOSINGBEHAVIOR = _descriptor.EnumDescriptor(
-  name='ClosingBehavior',
-  full_name='org.apache.beam.runner_api.v1.ClosingBehavior',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='EMIT_ALWAYS', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='EMIT_IF_NONEMPTY', index=1, number=1,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=7747,
-  serialized_end=7803,
-)
-_sym_db.RegisterEnumDescriptor(_CLOSINGBEHAVIOR)
-
-ClosingBehavior = enum_type_wrapper.EnumTypeWrapper(_CLOSINGBEHAVIOR)
-_OUTPUTTIME = _descriptor.EnumDescriptor(
-  name='OutputTime',
-  full_name='org.apache.beam.runner_api.v1.OutputTime',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='END_OF_WINDOW', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='LATEST_IN_PANE', index=1, number=1,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='EARLIEST_IN_PANE', index=2, number=2,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=7805,
-  serialized_end=7878,
-)
-_sym_db.RegisterEnumDescriptor(_OUTPUTTIME)
-
-OutputTime = enum_type_wrapper.EnumTypeWrapper(_OUTPUTTIME)
-_TIMEDOMAIN = _descriptor.EnumDescriptor(
-  name='TimeDomain',
-  full_name='org.apache.beam.runner_api.v1.TimeDomain',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='EVENT_TIME', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='PROCESSING_TIME', index=1, number=1,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='SYNCHRONIZED_PROCESSING_TIME', index=2, number=2,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=7880,
-  serialized_end=7963,
-)
-_sym_db.RegisterEnumDescriptor(_TIMEDOMAIN)
-
-TimeDomain = enum_type_wrapper.EnumTypeWrapper(_TIMEDOMAIN)
-BOUNDED = 0
-UNBOUNDED = 1
-NON_MERGING = 0
-NEEDS_MERGE = 1
-ALREADY_MERGED = 2
-DISCARDING = 0
-ACCUMULATING = 1
-EMIT_ALWAYS = 0
-EMIT_IF_NONEMPTY = 1
-END_OF_WINDOW = 0
-LATEST_IN_PANE = 1
-EARLIEST_IN_PANE = 2
-EVENT_TIME = 0
-PROCESSING_TIME = 1
-SYNCHRONIZED_PROCESSING_TIME = 2
-
-
-_PARAMETER_TYPE = _descriptor.EnumDescriptor(
-  name='Type',
-  full_name='org.apache.beam.runner_api.v1.Parameter.Type',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='WINDOW', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='PIPELINE_OPTIONS', index=1, number=1,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='RESTRICTION_TRACKER', index=2, number=2,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=3413,
-  serialized_end=3478,
-)
-_sym_db.RegisterEnumDescriptor(_PARAMETER_TYPE)
-
-_DISPLAYDATA_TYPE = _descriptor.EnumDescriptor(
-  name='Type',
-  full_name='org.apache.beam.runner_api.v1.DisplayData.Type',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='STRING', index=0, number=0,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='INTEGER', index=1, number=1,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='FLOAT', index=2, number=2,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='BOOLEAN', index=3, number=3,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='TIMESTAMP', index=4, number=4,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='DURATION', index=5, number=5,
-      options=None,
-      type=None),
-    _descriptor.EnumValueDescriptor(
-      name='JAVA_CLASS', index=6, number=6,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=7481,
-  serialized_end=7581,
-)
-_sym_db.RegisterEnumDescriptor(_DISPLAYDATA_TYPE)
-
-
-_COMPONENTS_TRANSFORMSENTRY = _descriptor.Descriptor(
-  name='TransformsEntry',
-  full_name='org.apache.beam.runner_api.v1.Components.TransformsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.Components.TransformsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.Components.TransformsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=512,
-  serialized_end=604,
-)
-
-_COMPONENTS_PCOLLECTIONSENTRY = _descriptor.Descriptor(
-  name='PcollectionsEntry',
-  full_name='org.apache.beam.runner_api.v1.Components.PcollectionsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.Components.PcollectionsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.Components.PcollectionsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=606,
-  serialized_end=701,
-)
-
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY = _descriptor.Descriptor(
-  name='WindowingStrategiesEntry',
-  full_name='org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=703,
-  serialized_end=811,
-)
-
-_COMPONENTS_CODERSENTRY = _descriptor.Descriptor(
-  name='CodersEntry',
-  full_name='org.apache.beam.runner_api.v1.Components.CodersEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.Components.CodersEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.Components.CodersEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=813,
-  serialized_end=896,
-)
-
-_COMPONENTS_ENVIRONMENTSENTRY = _descriptor.Descriptor(
-  name='EnvironmentsEntry',
-  full_name='org.apache.beam.runner_api.v1.Components.EnvironmentsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.Components.EnvironmentsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.Components.EnvironmentsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=898,
-  serialized_end=993,
-)
-
-_COMPONENTS = _descriptor.Descriptor(
-  name='Components',
-  full_name='org.apache.beam.runner_api.v1.Components',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='transforms', full_name='org.apache.beam.runner_api.v1.Components.transforms', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='pcollections', full_name='org.apache.beam.runner_api.v1.Components.pcollections', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='windowing_strategies', full_name='org.apache.beam.runner_api.v1.Components.windowing_strategies', index=2,
-      number=3, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='coders', full_name='org.apache.beam.runner_api.v1.Components.coders', index=3,
-      number=4, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='environments', full_name='org.apache.beam.runner_api.v1.Components.environments', index=4,
-      number=5, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_COMPONENTS_TRANSFORMSENTRY, _COMPONENTS_PCOLLECTIONSENTRY, _COMPONENTS_WINDOWINGSTRATEGIESENTRY, _COMPONENTS_CODERSENTRY, _COMPONENTS_ENVIRONMENTSENTRY, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=84,
-  serialized_end=993,
-)
-
-
-_MESSAGEWITHCOMPONENTS = _descriptor.Descriptor(
-  name='MessageWithComponents',
-  full_name='org.apache.beam.runner_api.v1.MessageWithComponents',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='components', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.components', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='coder', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.coder', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='combine_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.combine_payload', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='sdk_function_spec', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.sdk_function_spec', index=3,
-      number=4, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='par_do_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.par_do_payload', index=4,
-      number=6, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='ptransform', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.ptransform', index=5,
-      number=7, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='pcollection', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.pcollection', index=6,
-      number=8, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='read_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.read_payload', index=7,
-      number=9, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='side_input', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.side_input', index=8,
-      number=11, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='window_into_payload', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.window_into_payload', index=9,
-      number=12, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='windowing_strategy', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.windowing_strategy', index=10,
-      number=13, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='function_spec', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.function_spec', index=11,
-      number=14, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='root', full_name='org.apache.beam.runner_api.v1.MessageWithComponents.root',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=996,
-  serialized_end=1862,
-)
-
-
-_PIPELINE = _descriptor.Descriptor(
-  name='Pipeline',
-  full_name='org.apache.beam.runner_api.v1.Pipeline',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='components', full_name='org.apache.beam.runner_api.v1.Pipeline.components', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='root_transform_ids', full_name='org.apache.beam.runner_api.v1.Pipeline.root_transform_ids', index=1,
-      number=2, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='display_data', full_name='org.apache.beam.runner_api.v1.Pipeline.display_data', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=1865,
-  serialized_end=2032,
-)
-
-
-_PTRANSFORM_INPUTSENTRY = _descriptor.Descriptor(
-  name='InputsEntry',
-  full_name='org.apache.beam.runner_api.v1.PTransform.InputsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.PTransform.InputsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.PTransform.InputsEntry.value', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2362,
-  serialized_end=2407,
-)
-
-_PTRANSFORM_OUTPUTSENTRY = _descriptor.Descriptor(
-  name='OutputsEntry',
-  full_name='org.apache.beam.runner_api.v1.PTransform.OutputsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.PTransform.OutputsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.PTransform.OutputsEntry.value', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2409,
-  serialized_end=2455,
-)
-
-_PTRANSFORM = _descriptor.Descriptor(
-  name='PTransform',
-  full_name='org.apache.beam.runner_api.v1.PTransform',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='unique_name', full_name='org.apache.beam.runner_api.v1.PTransform.unique_name', index=0,
-      number=5, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='spec', full_name='org.apache.beam.runner_api.v1.PTransform.spec', index=1,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='subtransforms', full_name='org.apache.beam.runner_api.v1.PTransform.subtransforms', index=2,
-      number=2, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='inputs', full_name='org.apache.beam.runner_api.v1.PTransform.inputs', index=3,
-      number=3, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='outputs', full_name='org.apache.beam.runner_api.v1.PTransform.outputs', index=4,
-      number=4, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='display_data', full_name='org.apache.beam.runner_api.v1.PTransform.display_data', index=5,
-      number=6, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_PTRANSFORM_INPUTSENTRY, _PTRANSFORM_OUTPUTSENTRY, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2035,
-  serialized_end=2455,
-)
-
-
-_PCOLLECTION = _descriptor.Descriptor(
-  name='PCollection',
-  full_name='org.apache.beam.runner_api.v1.PCollection',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='unique_name', full_name='org.apache.beam.runner_api.v1.PCollection.unique_name', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='coder_id', full_name='org.apache.beam.runner_api.v1.PCollection.coder_id', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='is_bounded', full_name='org.apache.beam.runner_api.v1.PCollection.is_bounded', index=2,
-      number=3, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='windowing_strategy_id', full_name='org.apache.beam.runner_api.v1.PCollection.windowing_strategy_id', index=3,
-      number=4, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='display_data', full_name='org.apache.beam.runner_api.v1.PCollection.display_data', index=4,
-      number=5, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2458,
-  serialized_end=2669,
-)
-
-
-_PARDOPAYLOAD_SIDEINPUTSENTRY = _descriptor.Descriptor(
-  name='SideInputsEntry',
-  full_name='org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3059,
-  serialized_end=3150,
-)
-
-_PARDOPAYLOAD_STATESPECSENTRY = _descriptor.Descriptor(
-  name='StateSpecsEntry',
-  full_name='org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3152,
-  serialized_end=3243,
-)
-
-_PARDOPAYLOAD_TIMERSPECSENTRY = _descriptor.Descriptor(
-  name='TimerSpecsEntry',
-  full_name='org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3245,
-  serialized_end=3336,
-)
-
-_PARDOPAYLOAD = _descriptor.Descriptor(
-  name='ParDoPayload',
-  full_name='org.apache.beam.runner_api.v1.ParDoPayload',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='do_fn', full_name='org.apache.beam.runner_api.v1.ParDoPayload.do_fn', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='parameters', full_name='org.apache.beam.runner_api.v1.ParDoPayload.parameters', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='side_inputs', full_name='org.apache.beam.runner_api.v1.ParDoPayload.side_inputs', index=2,
-      number=3, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='state_specs', full_name='org.apache.beam.runner_api.v1.ParDoPayload.state_specs', index=3,
-      number=4, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='timer_specs', full_name='org.apache.beam.runner_api.v1.ParDoPayload.timer_specs', index=4,
-      number=5, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_PARDOPAYLOAD_SIDEINPUTSENTRY, _PARDOPAYLOAD_STATESPECSENTRY, _PARDOPAYLOAD_TIMERSPECSENTRY, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=2672,
-  serialized_end=3336,
-)
-
-
-_PARAMETER = _descriptor.Descriptor(
-  name='Parameter',
-  full_name='org.apache.beam.runner_api.v1.Parameter',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='type', full_name='org.apache.beam.runner_api.v1.Parameter.type', index=0,
-      number=1, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-    _PARAMETER_TYPE,
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3339,
-  serialized_end=3478,
-)
-
-
-_STATESPEC = _descriptor.Descriptor(
-  name='StateSpec',
-  full_name='org.apache.beam.runner_api.v1.StateSpec',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3480,
-  serialized_end=3491,
-)
-
-
-_TIMERSPEC = _descriptor.Descriptor(
-  name='TimerSpec',
-  full_name='org.apache.beam.runner_api.v1.TimerSpec',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3493,
-  serialized_end=3504,
-)
-
-
-_READPAYLOAD = _descriptor.Descriptor(
-  name='ReadPayload',
-  full_name='org.apache.beam.runner_api.v1.ReadPayload',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='source', full_name='org.apache.beam.runner_api.v1.ReadPayload.source', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='is_bounded', full_name='org.apache.beam.runner_api.v1.ReadPayload.is_bounded', index=1,
-      number=2, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3507,
-  serialized_end=3646,
-)
-
-
-_WINDOWINTOPAYLOAD = _descriptor.Descriptor(
-  name='WindowIntoPayload',
-  full_name='org.apache.beam.runner_api.v1.WindowIntoPayload',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='window_fn', full_name='org.apache.beam.runner_api.v1.WindowIntoPayload.window_fn', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3648,
-  serialized_end=3734,
-)
-
-
-_COMBINEPAYLOAD_SIDEINPUTSENTRY = _descriptor.Descriptor(
-  name='SideInputsEntry',
-  full_name='org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry.key', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry.value', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3059,
-  serialized_end=3150,
-)
-
-_COMBINEPAYLOAD = _descriptor.Descriptor(
-  name='CombinePayload',
-  full_name='org.apache.beam.runner_api.v1.CombinePayload',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='combine_fn', full_name='org.apache.beam.runner_api.v1.CombinePayload.combine_fn', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='accumulator_coder_id', full_name='org.apache.beam.runner_api.v1.CombinePayload.accumulator_coder_id', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='parameters', full_name='org.apache.beam.runner_api.v1.CombinePayload.parameters', index=2,
-      number=3, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='side_inputs', full_name='org.apache.beam.runner_api.v1.CombinePayload.side_inputs', index=3,
-      number=4, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_COMBINEPAYLOAD_SIDEINPUTSENTRY, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=3737,
-  serialized_end=4090,
-)
-
-
-_CODER = _descriptor.Descriptor(
-  name='Coder',
-  full_name='org.apache.beam.runner_api.v1.Coder',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='spec', full_name='org.apache.beam.runner_api.v1.Coder.spec', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='component_coder_ids', full_name='org.apache.beam.runner_api.v1.Coder.component_coder_ids', index=1,
-      number=2, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4092,
-  serialized_end=4190,
-)
-
-
-_WINDOWINGSTRATEGY = _descriptor.Descriptor(
-  name='WindowingStrategy',
-  full_name='org.apache.beam.runner_api.v1.WindowingStrategy',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='window_fn', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.window_fn', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='merge_status', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.merge_status', index=1,
-      number=2, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='window_coder_id', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.window_coder_id', index=2,
-      number=3, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='trigger', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.trigger', index=3,
-      number=4, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='accumulation_mode', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.accumulation_mode', index=4,
-      number=5, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='output_time', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.output_time', index=5,
-      number=6, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='closing_behavior', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.closing_behavior', index=6,
-      number=7, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='allowed_lateness', full_name='org.apache.beam.runner_api.v1.WindowingStrategy.allowed_lateness', index=7,
-      number=8, type=3, cpp_type=2, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=4193,
-  serialized_end=4667,
-)
-
-
-_TRIGGER_AFTERALL = _descriptor.Descriptor(
-  name='AfterAll',
-  full_name='org.apache.beam.runner_api.v1.Trigger.AfterAll',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='subtriggers', full_name='org.apache.beam.runner_api.v1.Trigger.AfterAll.subtriggers', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=5602,
-  serialized_end=5673,
-)
-
-_TRIGGER_AFTERANY = _descriptor.Descriptor(
-  name='AfterAny',
-  full_name='org.apache.beam.runner_api.v1.Trigger.AfterAny',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='subtriggers', full_name='org.apache.beam.runner_api.v1.Trigger.AfterAny.subtriggers', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=5675,
-  serialized_end=5746,
-)
-
-_TRIGGER_AFTEREACH = _descriptor.Descriptor(
-  name='AfterEach',
-  full_name='org.apache.beam.runner_api.v1.Trigger.AfterEach',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='subtriggers', full_name='org.apache.beam.runner_api.v1.Trigger.AfterEach.subtriggers', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=5748,
-  serialized_end=5820,
-)
-
-_TRIGGER_AFTERENDOFWINDOW = _descriptor.Descriptor(
-  name='AfterEndOfWindow',
-  full_name='org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='early_firings', full_name='org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow.early_firings', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='late_firings', full_name='org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow.late_firings', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=5823,
-  serialized_end=5966,
-)
-
-_TRIGGER_AFTERPROCESSINGTIME = _descriptor.Descriptor(
-  name='AfterProcessingTime',
-  full_name='org.apache.beam.runner_api.v1.Trigger.AfterProcessingTime',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='timestamp_transforms', full_name='org.apache.beam.runner_api.v1.Trigger.AfterProcessingTime.timestamp_transforms', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=5968,
-  serialized_end=6070,
-)
-
-_TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME = _descriptor.Descriptor(
-  name='AfterSynchronizedProcessingTime',
-  full_name='org.apache.beam.runner_api.v1.Trigger.AfterSynchronizedProcessingTime',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6072,
-  serialized_end=6105,
-)
-
-_TRIGGER_DEFAULT = _descriptor.Descriptor(
-  name='Default',
-  full_name='org.apache.beam.runner_api.v1.Trigger.Default',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6107,
-  serialized_end=6116,
-)
-
-_TRIGGER_ELEMENTCOUNT = _descriptor.Descriptor(
-  name='ElementCount',
-  full_name='org.apache.beam.runner_api.v1.Trigger.ElementCount',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='element_count', full_name='org.apache.beam.runner_api.v1.Trigger.ElementCount.element_count', index=0,
-      number=1, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6118,
-  serialized_end=6155,
-)
-
-_TRIGGER_NEVER = _descriptor.Descriptor(
-  name='Never',
-  full_name='org.apache.beam.runner_api.v1.Trigger.Never',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6157,
-  serialized_end=6164,
-)
-
-_TRIGGER_ALWAYS = _descriptor.Descriptor(
-  name='Always',
-  full_name='org.apache.beam.runner_api.v1.Trigger.Always',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6166,
-  serialized_end=6174,
-)
-
-_TRIGGER_ORFINALLY = _descriptor.Descriptor(
-  name='OrFinally',
-  full_name='org.apache.beam.runner_api.v1.Trigger.OrFinally',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='main', full_name='org.apache.beam.runner_api.v1.Trigger.OrFinally.main', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='finally', full_name='org.apache.beam.runner_api.v1.Trigger.OrFinally.finally', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6176,
-  serialized_end=6298,
-)
-
-_TRIGGER_REPEAT = _descriptor.Descriptor(
-  name='Repeat',
-  full_name='org.apache.beam.runner_api.v1.Trigger.Repeat',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='subtrigger', full_name='org.apache.beam.runner_api.v1.Trigger.Repeat.subtrigger', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6300,
-  serialized_end=6368,
-)
-
-_TRIGGER = _descriptor.Descriptor(
-  name='Trigger',
-  full_name='org.apache.beam.runner_api.v1.Trigger',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='after_all', full_name='org.apache.beam.runner_api.v1.Trigger.after_all', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='after_any', full_name='org.apache.beam.runner_api.v1.Trigger.after_any', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='after_each', full_name='org.apache.beam.runner_api.v1.Trigger.after_each', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='after_end_of_window', full_name='org.apache.beam.runner_api.v1.Trigger.after_end_of_window', index=3,
-      number=4, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='after_processing_time', full_name='org.apache.beam.runner_api.v1.Trigger.after_processing_time', index=4,
-      number=5, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='after_synchronized_processing_time', full_name='org.apache.beam.runner_api.v1.Trigger.after_synchronized_processing_time', index=5,
-      number=6, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='always', full_name='org.apache.beam.runner_api.v1.Trigger.always', index=6,
-      number=12, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='default', full_name='org.apache.beam.runner_api.v1.Trigger.default', index=7,
-      number=7, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='element_count', full_name='org.apache.beam.runner_api.v1.Trigger.element_count', index=8,
-      number=8, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='never', full_name='org.apache.beam.runner_api.v1.Trigger.never', index=9,
-      number=9, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='or_finally', full_name='org.apache.beam.runner_api.v1.Trigger.or_finally', index=10,
-      number=10, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='repeat', full_name='org.apache.beam.runner_api.v1.Trigger.repeat', index=11,
-      number=11, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_TRIGGER_AFTERALL, _TRIGGER_AFTERANY, _TRIGGER_AFTEREACH, _TRIGGER_AFTERENDOFWINDOW, _TRIGGER_AFTERPROCESSINGTIME, _TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME, _TRIGGER_DEFAULT, _TRIGGER_ELEMENTCOUNT, _TRIGGER_NEVER, _TRIGGER_ALWAYS, _TRIGGER_ORFINALLY, _TRIGGER_REPEAT, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='trigger', full_name='org.apache.beam.runner_api.v1.Trigger.trigger',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=4670,
-  serialized_end=6379,
-)
-
-
-_TIMESTAMPTRANSFORM_DELAY = _descriptor.Descriptor(
-  name='Delay',
-  full_name='org.apache.beam.runner_api.v1.TimestampTransform.Delay',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='delay_millis', full_name='org.apache.beam.runner_api.v1.TimestampTransform.Delay.delay_millis', index=0,
-      number=1, type=3, cpp_type=2, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6557,
-  serialized_end=6586,
-)
-
-_TIMESTAMPTRANSFORM_ALIGNTO = _descriptor.Descriptor(
-  name='AlignTo',
-  full_name='org.apache.beam.runner_api.v1.TimestampTransform.AlignTo',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='period', full_name='org.apache.beam.runner_api.v1.TimestampTransform.AlignTo.period', index=0,
-      number=3, type=3, cpp_type=2, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='offset', full_name='org.apache.beam.runner_api.v1.TimestampTransform.AlignTo.offset', index=1,
-      number=4, type=3, cpp_type=2, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6588,
-  serialized_end=6629,
-)
-
-_TIMESTAMPTRANSFORM = _descriptor.Descriptor(
-  name='TimestampTransform',
-  full_name='org.apache.beam.runner_api.v1.TimestampTransform',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='delay', full_name='org.apache.beam.runner_api.v1.TimestampTransform.delay', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='align_to', full_name='org.apache.beam.runner_api.v1.TimestampTransform.align_to', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_TIMESTAMPTRANSFORM_DELAY, _TIMESTAMPTRANSFORM_ALIGNTO, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-    _descriptor.OneofDescriptor(
-      name='timestamp_transform', full_name='org.apache.beam.runner_api.v1.TimestampTransform.timestamp_transform',
-      index=0, containing_type=None, fields=[]),
-  ],
-  serialized_start=6382,
-  serialized_end=6652,
-)
-
-
-_SIDEINPUT = _descriptor.Descriptor(
-  name='SideInput',
-  full_name='org.apache.beam.runner_api.v1.SideInput',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='access_pattern', full_name='org.apache.beam.runner_api.v1.SideInput.access_pattern', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='view_fn', full_name='org.apache.beam.runner_api.v1.SideInput.view_fn', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='window_mapping_fn', full_name='org.apache.beam.runner_api.v1.SideInput.window_mapping_fn', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6655,
-  serialized_end=6875,
-)
-
-
-_ENVIRONMENT = _descriptor.Descriptor(
-  name='Environment',
-  full_name='org.apache.beam.runner_api.v1.Environment',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='url', full_name='org.apache.beam.runner_api.v1.Environment.url', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6877,
-  serialized_end=6903,
-)
-
-
-_SDKFUNCTIONSPEC = _descriptor.Descriptor(
-  name='SdkFunctionSpec',
-  full_name='org.apache.beam.runner_api.v1.SdkFunctionSpec',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='spec', full_name='org.apache.beam.runner_api.v1.SdkFunctionSpec.spec', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='environment_id', full_name='org.apache.beam.runner_api.v1.SdkFunctionSpec.environment_id', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=6905,
-  serialized_end=7005,
-)
-
-
-_FUNCTIONSPEC = _descriptor.Descriptor(
-  name='FunctionSpec',
-  full_name='org.apache.beam.runner_api.v1.FunctionSpec',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='urn', full_name='org.apache.beam.runner_api.v1.FunctionSpec.urn', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='parameter', full_name='org.apache.beam.runner_api.v1.FunctionSpec.parameter', index=1,
-      number=2, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=7007,
-  serialized_end=7075,
-)
-
-
-_DISPLAYDATA_IDENTIFIER = _descriptor.Descriptor(
-  name='Identifier',
-  full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='transform_id', full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier.transform_id', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='transform_urn', full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier.transform_urn', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='key', full_name='org.apache.beam.runner_api.v1.DisplayData.Identifier.key', index=2,
-      number=3, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=7157,
-  serialized_end=7227,
-)
-
-_DISPLAYDATA_ITEM = _descriptor.Descriptor(
-  name='Item',
-  full_name='org.apache.beam.runner_api.v1.DisplayData.Item',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='id', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.id', index=0,
-      number=1, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='type', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.type', index=1,
-      number=2, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='value', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.value', index=2,
-      number=3, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='short_value', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.short_value', index=3,
-      number=4, type=11, cpp_type=10, label=1,
-      has_default_value=False, default_value=None,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='label', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.label', index=4,
-      number=5, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='link_url', full_name='org.apache.beam.runner_api.v1.DisplayData.Item.link_url', index=5,
-      number=6, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=7230,
-  serialized_end=7479,
-)
-
-_DISPLAYDATA = _descriptor.Descriptor(
-  name='DisplayData',
-  full_name='org.apache.beam.runner_api.v1.DisplayData',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='items', full_name='org.apache.beam.runner_api.v1.DisplayData.items', index=0,
-      number=1, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_DISPLAYDATA_IDENTIFIER, _DISPLAYDATA_ITEM, ],
-  enum_types=[
-    _DISPLAYDATA_TYPE,
-  ],
-  options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=7078,
-  serialized_end=7581,
-)
-
-_COMPONENTS_TRANSFORMSENTRY.fields_by_name['value'].message_type = _PTRANSFORM
-_COMPONENTS_TRANSFORMSENTRY.containing_type = _COMPONENTS
-_COMPONENTS_PCOLLECTIONSENTRY.fields_by_name['value'].message_type = _PCOLLECTION
-_COMPONENTS_PCOLLECTIONSENTRY.containing_type = _COMPONENTS
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY.fields_by_name['value'].message_type = _WINDOWINGSTRATEGY
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY.containing_type = _COMPONENTS
-_COMPONENTS_CODERSENTRY.fields_by_name['value'].message_type = _CODER
-_COMPONENTS_CODERSENTRY.containing_type = _COMPONENTS
-_COMPONENTS_ENVIRONMENTSENTRY.fields_by_name['value'].message_type = _ENVIRONMENT
-_COMPONENTS_ENVIRONMENTSENTRY.containing_type = _COMPONENTS
-_COMPONENTS.fields_by_name['transforms'].message_type = _COMPONENTS_TRANSFORMSENTRY
-_COMPONENTS.fields_by_name['pcollections'].message_type = _COMPONENTS_PCOLLECTIONSENTRY
-_COMPONENTS.fields_by_name['windowing_strategies'].message_type = _COMPONENTS_WINDOWINGSTRATEGIESENTRY
-_COMPONENTS.fields_by_name['coders'].message_type = _COMPONENTS_CODERSENTRY
-_COMPONENTS.fields_by_name['environments'].message_type = _COMPONENTS_ENVIRONMENTSENTRY
-_MESSAGEWITHCOMPONENTS.fields_by_name['components'].message_type = _COMPONENTS
-_MESSAGEWITHCOMPONENTS.fields_by_name['coder'].message_type = _CODER
-_MESSAGEWITHCOMPONENTS.fields_by_name['combine_payload'].message_type = _COMBINEPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['sdk_function_spec'].message_type = _SDKFUNCTIONSPEC
-_MESSAGEWITHCOMPONENTS.fields_by_name['par_do_payload'].message_type = _PARDOPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['ptransform'].message_type = _PTRANSFORM
-_MESSAGEWITHCOMPONENTS.fields_by_name['pcollection'].message_type = _PCOLLECTION
-_MESSAGEWITHCOMPONENTS.fields_by_name['read_payload'].message_type = _READPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['side_input'].message_type = _SIDEINPUT
-_MESSAGEWITHCOMPONENTS.fields_by_name['window_into_payload'].message_type = _WINDOWINTOPAYLOAD
-_MESSAGEWITHCOMPONENTS.fields_by_name['windowing_strategy'].message_type = _WINDOWINGSTRATEGY
-_MESSAGEWITHCOMPONENTS.fields_by_name['function_spec'].message_type = _FUNCTIONSPEC
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['coder'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['coder'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['combine_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['combine_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['sdk_function_spec'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['sdk_function_spec'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['par_do_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['par_do_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['ptransform'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['ptransform'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['pcollection'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['pcollection'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['read_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['read_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['side_input'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['side_input'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['window_into_payload'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['window_into_payload'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['windowing_strategy'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['windowing_strategy'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_MESSAGEWITHCOMPONENTS.oneofs_by_name['root'].fields.append(
-  _MESSAGEWITHCOMPONENTS.fields_by_name['function_spec'])
-_MESSAGEWITHCOMPONENTS.fields_by_name['function_spec'].containing_oneof = _MESSAGEWITHCOMPONENTS.oneofs_by_name['root']
-_PIPELINE.fields_by_name['components'].message_type = _COMPONENTS
-_PIPELINE.fields_by_name['display_data'].message_type = _DISPLAYDATA
-_PTRANSFORM_INPUTSENTRY.containing_type = _PTRANSFORM
-_PTRANSFORM_OUTPUTSENTRY.containing_type = _PTRANSFORM
-_PTRANSFORM.fields_by_name['spec'].message_type = _FUNCTIONSPEC
-_PTRANSFORM.fields_by_name['inputs'].message_type = _PTRANSFORM_INPUTSENTRY
-_PTRANSFORM.fields_by_name['outputs'].message_type = _PTRANSFORM_OUTPUTSENTRY
-_PTRANSFORM.fields_by_name['display_data'].message_type = _DISPLAYDATA
-_PCOLLECTION.fields_by_name['is_bounded'].enum_type = _ISBOUNDED
-_PCOLLECTION.fields_by_name['display_data'].message_type = _DISPLAYDATA
-_PARDOPAYLOAD_SIDEINPUTSENTRY.fields_by_name['value'].message_type = _SIDEINPUT
-_PARDOPAYLOAD_SIDEINPUTSENTRY.containing_type = _PARDOPAYLOAD
-_PARDOPAYLOAD_STATESPECSENTRY.fields_by_name['value'].message_type = _STATESPEC
-_PARDOPAYLOAD_STATESPECSENTRY.containing_type = _PARDOPAYLOAD
-_PARDOPAYLOAD_TIMERSPECSENTRY.fields_by_name['value'].message_type = _TIMERSPEC
-_PARDOPAYLOAD_TIMERSPECSENTRY.containing_type = _PARDOPAYLOAD
-_PARDOPAYLOAD.fields_by_name['do_fn'].message_type = _SDKFUNCTIONSPEC
-_PARDOPAYLOAD.fields_by_name['parameters'].message_type = _PARAMETER
-_PARDOPAYLOAD.fields_by_name['side_inputs'].message_type = _PARDOPAYLOAD_SIDEINPUTSENTRY
-_PARDOPAYLOAD.fields_by_name['state_specs'].message_type = _PARDOPAYLOAD_STATESPECSENTRY
-_PARDOPAYLOAD.fields_by_name['timer_specs'].message_type = _PARDOPAYLOAD_TIMERSPECSENTRY
-_PARAMETER.fields_by_name['type'].enum_type = _PARAMETER_TYPE
-_PARAMETER_TYPE.containing_type = _PARAMETER
-_READPAYLOAD.fields_by_name['source'].message_type = _SDKFUNCTIONSPEC
-_READPAYLOAD.fields_by_name['is_bounded'].enum_type = _ISBOUNDED
-_WINDOWINTOPAYLOAD.fields_by_name['window_fn'].message_type = _SDKFUNCTIONSPEC
-_COMBINEPAYLOAD_SIDEINPUTSENTRY.fields_by_name['value'].message_type = _SIDEINPUT
-_COMBINEPAYLOAD_SIDEINPUTSENTRY.containing_type = _COMBINEPAYLOAD
-_COMBINEPAYLOAD.fields_by_name['combine_fn'].message_type = _SDKFUNCTIONSPEC
-_COMBINEPAYLOAD.fields_by_name['parameters'].message_type = _PARAMETER
-_COMBINEPAYLOAD.fields_by_name['side_inputs'].message_type = _COMBINEPAYLOAD_SIDEINPUTSENTRY
-_CODER.fields_by_name['spec'].message_type = _SDKFUNCTIONSPEC
-_WINDOWINGSTRATEGY.fields_by_name['window_fn'].message_type = _SDKFUNCTIONSPEC
-_WINDOWINGSTRATEGY.fields_by_name['merge_status'].enum_type = _MERGESTATUS
-_WINDOWINGSTRATEGY.fields_by_name['trigger'].message_type = _TRIGGER
-_WINDOWINGSTRATEGY.fields_by_name['accumulation_mode'].enum_type = _ACCUMULATIONMODE
-_WINDOWINGSTRATEGY.fields_by_name['output_time'].enum_type = _OUTPUTTIME
-_WINDOWINGSTRATEGY.fields_by_name['closing_behavior'].enum_type = _CLOSINGBEHAVIOR
-_TRIGGER_AFTERALL.fields_by_name['subtriggers'].message_type = _TRIGGER
-_TRIGGER_AFTERALL.containing_type = _TRIGGER
-_TRIGGER_AFTERANY.fields_by_name['subtriggers'].message_type = _TRIGGER
-_TRIGGER_AFTERANY.containing_type = _TRIGGER
-_TRIGGER_AFTEREACH.fields_by_name['subtriggers'].message_type = _TRIGGER
-_TRIGGER_AFTEREACH.containing_type = _TRIGGER
-_TRIGGER_AFTERENDOFWINDOW.fields_by_name['early_firings'].message_type = _TRIGGER
-_TRIGGER_AFTERENDOFWINDOW.fields_by_name['late_firings'].message_type = _TRIGGER
-_TRIGGER_AFTERENDOFWINDOW.containing_type = _TRIGGER
-_TRIGGER_AFTERPROCESSINGTIME.fields_by_name['timestamp_transforms'].message_type = _TIMESTAMPTRANSFORM
-_TRIGGER_AFTERPROCESSINGTIME.containing_type = _TRIGGER
-_TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME.containing_type = _TRIGGER
-_TRIGGER_DEFAULT.containing_type = _TRIGGER
-_TRIGGER_ELEMENTCOUNT.containing_type = _TRIGGER
-_TRIGGER_NEVER.containing_type = _TRIGGER
-_TRIGGER_ALWAYS.containing_type = _TRIGGER
-_TRIGGER_ORFINALLY.fields_by_name['main'].message_type = _TRIGGER
-_TRIGGER_ORFINALLY.fields_by_name['finally'].message_type = _TRIGGER
-_TRIGGER_ORFINALLY.containing_type = _TRIGGER
-_TRIGGER_REPEAT.fields_by_name['subtrigger'].message_type = _TRIGGER
-_TRIGGER_REPEAT.containing_type = _TRIGGER
-_TRIGGER.fields_by_name['after_all'].message_type = _TRIGGER_AFTERALL
-_TRIGGER.fields_by_name['after_any'].message_type = _TRIGGER_AFTERANY
-_TRIGGER.fields_by_name['after_each'].message_type = _TRIGGER_AFTEREACH
-_TRIGGER.fields_by_name['after_end_of_window'].message_type = _TRIGGER_AFTERENDOFWINDOW
-_TRIGGER.fields_by_name['after_processing_time'].message_type = _TRIGGER_AFTERPROCESSINGTIME
-_TRIGGER.fields_by_name['after_synchronized_processing_time'].message_type = _TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME
-_TRIGGER.fields_by_name['always'].message_type = _TRIGGER_ALWAYS
-_TRIGGER.fields_by_name['default'].message_type = _TRIGGER_DEFAULT
-_TRIGGER.fields_by_name['element_count'].message_type = _TRIGGER_ELEMENTCOUNT
-_TRIGGER.fields_by_name['never'].message_type = _TRIGGER_NEVER
-_TRIGGER.fields_by_name['or_finally'].message_type = _TRIGGER_ORFINALLY
-_TRIGGER.fields_by_name['repeat'].message_type = _TRIGGER_REPEAT
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['after_all'])
-_TRIGGER.fields_by_name['after_all'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['after_any'])
-_TRIGGER.fields_by_name['after_any'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['after_each'])
-_TRIGGER.fields_by_name['after_each'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['after_end_of_window'])
-_TRIGGER.fields_by_name['after_end_of_window'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['after_processing_time'])
-_TRIGGER.fields_by_name['after_processing_time'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['after_synchronized_processing_time'])
-_TRIGGER.fields_by_name['after_synchronized_processing_time'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['always'])
-_TRIGGER.fields_by_name['always'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['default'])
-_TRIGGER.fields_by_name['default'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['element_count'])
-_TRIGGER.fields_by_name['element_count'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['never'])
-_TRIGGER.fields_by_name['never'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['or_finally'])
-_TRIGGER.fields_by_name['or_finally'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TRIGGER.oneofs_by_name['trigger'].fields.append(
-  _TRIGGER.fields_by_name['repeat'])
-_TRIGGER.fields_by_name['repeat'].containing_oneof = _TRIGGER.oneofs_by_name['trigger']
-_TIMESTAMPTRANSFORM_DELAY.containing_type = _TIMESTAMPTRANSFORM
-_TIMESTAMPTRANSFORM_ALIGNTO.containing_type = _TIMESTAMPTRANSFORM
-_TIMESTAMPTRANSFORM.fields_by_name['delay'].message_type = _TIMESTAMPTRANSFORM_DELAY
-_TIMESTAMPTRANSFORM.fields_by_name['align_to'].message_type = _TIMESTAMPTRANSFORM_ALIGNTO
-_TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform'].fields.append(
-  _TIMESTAMPTRANSFORM.fields_by_name['delay'])
-_TIMESTAMPTRANSFORM.fields_by_name['delay'].containing_oneof = _TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform']
-_TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform'].fields.append(
-  _TIMESTAMPTRANSFORM.fields_by_name['align_to'])
-_TIMESTAMPTRANSFORM.fields_by_name['align_to'].containing_oneof = _TIMESTAMPTRANSFORM.oneofs_by_name['timestamp_transform']
-_SIDEINPUT.fields_by_name['access_pattern'].message_type = _FUNCTIONSPEC
-_SIDEINPUT.fields_by_name['view_fn'].message_type = _SDKFUNCTIONSPEC
-_SIDEINPUT.fields_by_name['window_mapping_fn'].message_type = _SDKFUNCTIONSPEC
-_SDKFUNCTIONSPEC.fields_by_name['spec'].message_type = _FUNCTIONSPEC
-_FUNCTIONSPEC.fields_by_name['parameter'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_DISPLAYDATA_IDENTIFIER.containing_type = _DISPLAYDATA
-_DISPLAYDATA_ITEM.fields_by_name['id'].message_type = _DISPLAYDATA_IDENTIFIER
-_DISPLAYDATA_ITEM.fields_by_name['type'].enum_type = _DISPLAYDATA_TYPE
-_DISPLAYDATA_ITEM.fields_by_name['value'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_DISPLAYDATA_ITEM.fields_by_name['short_value'].message_type = google_dot_protobuf_dot_any__pb2._ANY
-_DISPLAYDATA_ITEM.containing_type = _DISPLAYDATA
-_DISPLAYDATA.fields_by_name['items'].message_type = _DISPLAYDATA_ITEM
-_DISPLAYDATA_TYPE.containing_type = _DISPLAYDATA
-DESCRIPTOR.message_types_by_name['Components'] = _COMPONENTS
-DESCRIPTOR.message_types_by_name['MessageWithComponents'] = _MESSAGEWITHCOMPONENTS
-DESCRIPTOR.message_types_by_name['Pipeline'] = _PIPELINE
-DESCRIPTOR.message_types_by_name['PTransform'] = _PTRANSFORM
-DESCRIPTOR.message_types_by_name['PCollection'] = _PCOLLECTION
-DESCRIPTOR.message_types_by_name['ParDoPayload'] = _PARDOPAYLOAD
-DESCRIPTOR.message_types_by_name['Parameter'] = _PARAMETER
-DESCRIPTOR.message_types_by_name['StateSpec'] = _STATESPEC
-DESCRIPTOR.message_types_by_name['TimerSpec'] = _TIMERSPEC
-DESCRIPTOR.message_types_by_name['ReadPayload'] = _READPAYLOAD
-DESCRIPTOR.message_types_by_name['WindowIntoPayload'] = _WINDOWINTOPAYLOAD
-DESCRIPTOR.message_types_by_name['CombinePayload'] = _COMBINEPAYLOAD
-DESCRIPTOR.message_types_by_name['Coder'] = _CODER
-DESCRIPTOR.message_types_by_name['WindowingStrategy'] = _WINDOWINGSTRATEGY
-DESCRIPTOR.message_types_by_name['Trigger'] = _TRIGGER
-DESCRIPTOR.message_types_by_name['TimestampTransform'] = _TIMESTAMPTRANSFORM
-DESCRIPTOR.message_types_by_name['SideInput'] = _SIDEINPUT
-DESCRIPTOR.message_types_by_name['Environment'] = _ENVIRONMENT
-DESCRIPTOR.message_types_by_name['SdkFunctionSpec'] = _SDKFUNCTIONSPEC
-DESCRIPTOR.message_types_by_name['FunctionSpec'] = _FUNCTIONSPEC
-DESCRIPTOR.message_types_by_name['DisplayData'] = _DISPLAYDATA
-DESCRIPTOR.enum_types_by_name['IsBounded'] = _ISBOUNDED
-DESCRIPTOR.enum_types_by_name['MergeStatus'] = _MERGESTATUS
-DESCRIPTOR.enum_types_by_name['AccumulationMode'] = _ACCUMULATIONMODE
-DESCRIPTOR.enum_types_by_name['ClosingBehavior'] = _CLOSINGBEHAVIOR
-DESCRIPTOR.enum_types_by_name['OutputTime'] = _OUTPUTTIME
-DESCRIPTOR.enum_types_by_name['TimeDomain'] = _TIMEDOMAIN
-
-Components = _reflection.GeneratedProtocolMessageType('Components', (_message.Message,), dict(
-
-  TransformsEntry = _reflection.GeneratedProtocolMessageType('TransformsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _COMPONENTS_TRANSFORMSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.TransformsEntry)
-    ))
-  ,
-
-  PcollectionsEntry = _reflection.GeneratedProtocolMessageType('PcollectionsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _COMPONENTS_PCOLLECTIONSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.PcollectionsEntry)
-    ))
-  ,
-
-  WindowingStrategiesEntry = _reflection.GeneratedProtocolMessageType('WindowingStrategiesEntry', (_message.Message,), dict(
-    DESCRIPTOR = _COMPONENTS_WINDOWINGSTRATEGIESENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.WindowingStrategiesEntry)
-    ))
-  ,
-
-  CodersEntry = _reflection.GeneratedProtocolMessageType('CodersEntry', (_message.Message,), dict(
-    DESCRIPTOR = _COMPONENTS_CODERSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.CodersEntry)
-    ))
-  ,
-
-  EnvironmentsEntry = _reflection.GeneratedProtocolMessageType('EnvironmentsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _COMPONENTS_ENVIRONMENTSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components.EnvironmentsEntry)
-    ))
-  ,
-  DESCRIPTOR = _COMPONENTS,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Components)
-  ))
-_sym_db.RegisterMessage(Components)
-_sym_db.RegisterMessage(Components.TransformsEntry)
-_sym_db.RegisterMessage(Components.PcollectionsEntry)
-_sym_db.RegisterMessage(Components.WindowingStrategiesEntry)
-_sym_db.RegisterMessage(Components.CodersEntry)
-_sym_db.RegisterMessage(Components.EnvironmentsEntry)
-
-MessageWithComponents = _reflection.GeneratedProtocolMessageType('MessageWithComponents', (_message.Message,), dict(
-  DESCRIPTOR = _MESSAGEWITHCOMPONENTS,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.MessageWithComponents)
-  ))
-_sym_db.RegisterMessage(MessageWithComponents)
-
-Pipeline = _reflection.GeneratedProtocolMessageType('Pipeline', (_message.Message,), dict(
-  DESCRIPTOR = _PIPELINE,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Pipeline)
-  ))
-_sym_db.RegisterMessage(Pipeline)
-
-PTransform = _reflection.GeneratedProtocolMessageType('PTransform', (_message.Message,), dict(
-
-  InputsEntry = _reflection.GeneratedProtocolMessageType('InputsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PTRANSFORM_INPUTSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PTransform.InputsEntry)
-    ))
-  ,
-
-  OutputsEntry = _reflection.GeneratedProtocolMessageType('OutputsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PTRANSFORM_OUTPUTSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PTransform.OutputsEntry)
-    ))
-  ,
-  DESCRIPTOR = _PTRANSFORM,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PTransform)
-  ))
-_sym_db.RegisterMessage(PTransform)
-_sym_db.RegisterMessage(PTransform.InputsEntry)
-_sym_db.RegisterMessage(PTransform.OutputsEntry)
-
-PCollection = _reflection.GeneratedProtocolMessageType('PCollection', (_message.Message,), dict(
-  DESCRIPTOR = _PCOLLECTION,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.PCollection)
-  ))
-_sym_db.RegisterMessage(PCollection)
-
-ParDoPayload = _reflection.GeneratedProtocolMessageType('ParDoPayload', (_message.Message,), dict(
-
-  SideInputsEntry = _reflection.GeneratedProtocolMessageType('SideInputsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PARDOPAYLOAD_SIDEINPUTSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload.SideInputsEntry)
-    ))
-  ,
-
-  StateSpecsEntry = _reflection.GeneratedProtocolMessageType('StateSpecsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PARDOPAYLOAD_STATESPECSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload.StateSpecsEntry)
-    ))
-  ,
-
-  TimerSpecsEntry = _reflection.GeneratedProtocolMessageType('TimerSpecsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _PARDOPAYLOAD_TIMERSPECSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload.TimerSpecsEntry)
-    ))
-  ,
-  DESCRIPTOR = _PARDOPAYLOAD,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ParDoPayload)
-  ))
-_sym_db.RegisterMessage(ParDoPayload)
-_sym_db.RegisterMessage(ParDoPayload.SideInputsEntry)
-_sym_db.RegisterMessage(ParDoPayload.StateSpecsEntry)
-_sym_db.RegisterMessage(ParDoPayload.TimerSpecsEntry)
-
-Parameter = _reflection.GeneratedProtocolMessageType('Parameter', (_message.Message,), dict(
-  DESCRIPTOR = _PARAMETER,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Parameter)
-  ))
-_sym_db.RegisterMessage(Parameter)
-
-StateSpec = _reflection.GeneratedProtocolMessageType('StateSpec', (_message.Message,), dict(
-  DESCRIPTOR = _STATESPEC,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.StateSpec)
-  ))
-_sym_db.RegisterMessage(StateSpec)
-
-TimerSpec = _reflection.GeneratedProtocolMessageType('TimerSpec', (_message.Message,), dict(
-  DESCRIPTOR = _TIMERSPEC,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimerSpec)
-  ))
-_sym_db.RegisterMessage(TimerSpec)
-
-ReadPayload = _reflection.GeneratedProtocolMessageType('ReadPayload', (_message.Message,), dict(
-  DESCRIPTOR = _READPAYLOAD,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.ReadPayload)
-  ))
-_sym_db.RegisterMessage(ReadPayload)
-
-WindowIntoPayload = _reflection.GeneratedProtocolMessageType('WindowIntoPayload', (_message.Message,), dict(
-  DESCRIPTOR = _WINDOWINTOPAYLOAD,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.WindowIntoPayload)
-  ))
-_sym_db.RegisterMessage(WindowIntoPayload)
-
-CombinePayload = _reflection.GeneratedProtocolMessageType('CombinePayload', (_message.Message,), dict(
-
-  SideInputsEntry = _reflection.GeneratedProtocolMessageType('SideInputsEntry', (_message.Message,), dict(
-    DESCRIPTOR = _COMBINEPAYLOAD_SIDEINPUTSENTRY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.CombinePayload.SideInputsEntry)
-    ))
-  ,
-  DESCRIPTOR = _COMBINEPAYLOAD,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.CombinePayload)
-  ))
-_sym_db.RegisterMessage(CombinePayload)
-_sym_db.RegisterMessage(CombinePayload.SideInputsEntry)
-
-Coder = _reflection.GeneratedProtocolMessageType('Coder', (_message.Message,), dict(
-  DESCRIPTOR = _CODER,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Coder)
-  ))
-_sym_db.RegisterMessage(Coder)
-
-WindowingStrategy = _reflection.GeneratedProtocolMessageType('WindowingStrategy', (_message.Message,), dict(
-  DESCRIPTOR = _WINDOWINGSTRATEGY,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.WindowingStrategy)
-  ))
-_sym_db.RegisterMessage(WindowingStrategy)
-
-Trigger = _reflection.GeneratedProtocolMessageType('Trigger', (_message.Message,), dict(
-
-  AfterAll = _reflection.GeneratedProtocolMessageType('AfterAll', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_AFTERALL,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterAll)
-    ))
-  ,
-
-  AfterAny = _reflection.GeneratedProtocolMessageType('AfterAny', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_AFTERANY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterAny)
-    ))
-  ,
-
-  AfterEach = _reflection.GeneratedProtocolMessageType('AfterEach', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_AFTEREACH,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterEach)
-    ))
-  ,
-
-  AfterEndOfWindow = _reflection.GeneratedProtocolMessageType('AfterEndOfWindow', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_AFTERENDOFWINDOW,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterEndOfWindow)
-    ))
-  ,
-
-  AfterProcessingTime = _reflection.GeneratedProtocolMessageType('AfterProcessingTime', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_AFTERPROCESSINGTIME,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterProcessingTime)
-    ))
-  ,
-
-  AfterSynchronizedProcessingTime = _reflection.GeneratedProtocolMessageType('AfterSynchronizedProcessingTime', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_AFTERSYNCHRONIZEDPROCESSINGTIME,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.AfterSynchronizedProcessingTime)
-    ))
-  ,
-
-  Default = _reflection.GeneratedProtocolMessageType('Default', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_DEFAULT,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Default)
-    ))
-  ,
-
-  ElementCount = _reflection.GeneratedProtocolMessageType('ElementCount', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_ELEMENTCOUNT,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.ElementCount)
-    ))
-  ,
-
-  Never = _reflection.GeneratedProtocolMessageType('Never', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_NEVER,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Never)
-    ))
-  ,
-
-  Always = _reflection.GeneratedProtocolMessageType('Always', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_ALWAYS,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Always)
-    ))
-  ,
-
-  OrFinally = _reflection.GeneratedProtocolMessageType('OrFinally', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_ORFINALLY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.OrFinally)
-    ))
-  ,
-
-  Repeat = _reflection.GeneratedProtocolMessageType('Repeat', (_message.Message,), dict(
-    DESCRIPTOR = _TRIGGER_REPEAT,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger.Repeat)
-    ))
-  ,
-  DESCRIPTOR = _TRIGGER,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Trigger)
-  ))
-_sym_db.RegisterMessage(Trigger)
-_sym_db.RegisterMessage(Trigger.AfterAll)
-_sym_db.RegisterMessage(Trigger.AfterAny)
-_sym_db.RegisterMessage(Trigger.AfterEach)
-_sym_db.RegisterMessage(Trigger.AfterEndOfWindow)
-_sym_db.RegisterMessage(Trigger.AfterProcessingTime)
-_sym_db.RegisterMessage(Trigger.AfterSynchronizedProcessingTime)
-_sym_db.RegisterMessage(Trigger.Default)
-_sym_db.RegisterMessage(Trigger.ElementCount)
-_sym_db.RegisterMessage(Trigger.Never)
-_sym_db.RegisterMessage(Trigger.Always)
-_sym_db.RegisterMessage(Trigger.OrFinally)
-_sym_db.RegisterMessage(Trigger.Repeat)
-
-TimestampTransform = _reflection.GeneratedProtocolMessageType('TimestampTransform', (_message.Message,), dict(
-
-  Delay = _reflection.GeneratedProtocolMessageType('Delay', (_message.Message,), dict(
-    DESCRIPTOR = _TIMESTAMPTRANSFORM_DELAY,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimestampTransform.Delay)
-    ))
-  ,
-
-  AlignTo = _reflection.GeneratedProtocolMessageType('AlignTo', (_message.Message,), dict(
-    DESCRIPTOR = _TIMESTAMPTRANSFORM_ALIGNTO,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimestampTransform.AlignTo)
-    ))
-  ,
-  DESCRIPTOR = _TIMESTAMPTRANSFORM,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.TimestampTransform)
-  ))
-_sym_db.RegisterMessage(TimestampTransform)
-_sym_db.RegisterMessage(TimestampTransform.Delay)
-_sym_db.RegisterMessage(TimestampTransform.AlignTo)
-
-SideInput = _reflection.GeneratedProtocolMessageType('SideInput', (_message.Message,), dict(
-  DESCRIPTOR = _SIDEINPUT,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.SideInput)
-  ))
-_sym_db.RegisterMessage(SideInput)
-
-Environment = _reflection.GeneratedProtocolMessageType('Environment', (_message.Message,), dict(
-  DESCRIPTOR = _ENVIRONMENT,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.Environment)
-  ))
-_sym_db.RegisterMessage(Environment)
-
-SdkFunctionSpec = _reflection.GeneratedProtocolMessageType('SdkFunctionSpec', (_message.Message,), dict(
-  DESCRIPTOR = _SDKFUNCTIONSPEC,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.SdkFunctionSpec)
-  ))
-_sym_db.RegisterMessage(SdkFunctionSpec)
-
-FunctionSpec = _reflection.GeneratedProtocolMessageType('FunctionSpec', (_message.Message,), dict(
-  DESCRIPTOR = _FUNCTIONSPEC,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.FunctionSpec)
-  ))
-_sym_db.RegisterMessage(FunctionSpec)
-
-DisplayData = _reflection.GeneratedProtocolMessageType('DisplayData', (_message.Message,), dict(
-
-  Identifier = _reflection.GeneratedProtocolMessageType('Identifier', (_message.Message,), dict(
-    DESCRIPTOR = _DISPLAYDATA_IDENTIFIER,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.DisplayData.Identifier)
-    ))
-  ,
-
-  Item = _reflection.GeneratedProtocolMessageType('Item', (_message.Message,), dict(
-    DESCRIPTOR = _DISPLAYDATA_ITEM,
-    __module__ = 'beam_runner_api_pb2'
-    # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.DisplayData.Item)
-    ))
-  ,
-  DESCRIPTOR = _DISPLAYDATA,
-  __module__ = 'beam_runner_api_pb2'
-  # @@protoc_insertion_point(class_scope:org.apache.beam.runner_api.v1.DisplayData)
-  ))
-_sym_db.RegisterMessage(DisplayData)
-_sym_db.RegisterMessage(DisplayData.Identifier)
-_sym_db.RegisterMessage(DisplayData.Item)
-
-
-DESCRIPTOR.has_options = True
-DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n$org.apache.beam.sdk.common.runner.v1B\tRunnerApi'))
-_COMPONENTS_TRANSFORMSENTRY.has_options = True
-_COMPONENTS_TRANSFORMSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_PCOLLECTIONSENTRY.has_options = True
-_COMPONENTS_PCOLLECTIONSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY.has_options = True
-_COMPONENTS_WINDOWINGSTRATEGIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_CODERSENTRY.has_options = True
-_COMPONENTS_CODERSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMPONENTS_ENVIRONMENTSENTRY.has_options = True
-_COMPONENTS_ENVIRONMENTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PTRANSFORM_INPUTSENTRY.has_options = True
-_PTRANSFORM_INPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PTRANSFORM_OUTPUTSENTRY.has_options = True
-_PTRANSFORM_OUTPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PARDOPAYLOAD_SIDEINPUTSENTRY.has_options = True
-_PARDOPAYLOAD_SIDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PARDOPAYLOAD_STATESPECSENTRY.has_options = True
-_PARDOPAYLOAD_STATESPECSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_PARDOPAYLOAD_TIMERSPECSENTRY.has_options = True
-_PARDOPAYLOAD_TIMERSPECSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-_COMBINEPAYLOAD_SIDEINPUTSENTRY.has_options = True
-_COMBINEPAYLOAD_SIDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
-# @@protoc_insertion_point(module_scope)

From e9cb40dc37683effb2ded1bfaab6dfc04ade698b Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@google.com>
Date: Wed, 17 May 2017 14:25:08 -0700
Subject: [PATCH 325/387] A couple of worker fixes.

---
 .../apache_beam/runners/worker/operation_specs.py     | 11 ++++++-----
 sdks/python/apache_beam/runners/worker/operations.py  |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/sdks/python/apache_beam/runners/worker/operation_specs.py b/sdks/python/apache_beam/runners/worker/operation_specs.py
index c03d9a25ec5f..db5eb765598b 100644
--- a/sdks/python/apache_beam/runners/worker/operation_specs.py
+++ b/sdks/python/apache_beam/runners/worker/operation_specs.py
@@ -305,7 +305,8 @@ def get_coder_from_spec(coder_spec):
   assert coder_spec is not None
 
   # Ignore the wrappers in these encodings.
-  # TODO(silviuc): Make sure with all the renamings that names below are ok.
+  ignored_wrappers = (
+      'com.google.cloud.dataflow.sdk.util.TimerOrElement$TimerOrElementCoder')
   if coder_spec['@type'] in ignored_wrappers:
     assert len(coder_spec['component_encodings']) == 1
     coder_spec = coder_spec['component_encodings'][0]
@@ -328,20 +329,20 @@ def get_coder_from_spec(coder_spec):
     return coders.WindowedValueCoder(value_coder, window_coder=window_coder)
   elif coder_spec['@type'] == 'kind:interval_window':
     assert ('component_encodings' not in coder_spec
-            or len(coder_spec['component_encodings'] == 0))
+            or not coder_spec['component_encodings'])
     return coders.IntervalWindowCoder()
   elif coder_spec['@type'] == 'kind:global_window':
     assert ('component_encodings' not in coder_spec
             or not coder_spec['component_encodings'])
-    return coders.GlobalWindowCoder()
+    return coders.coders.GlobalWindowCoder()
   elif coder_spec['@type'] == 'kind:length_prefix':
     assert len(coder_spec['component_encodings']) == 1
-    return coders.LengthPrefixCoder(
+    return coders.coders.LengthPrefixCoder(
         get_coder_from_spec(coder_spec['component_encodings'][0]))
 
   # We pass coders in the form "<coder_name>$<pickled_data>" to make the job
   # description JSON more readable.
-  return coders.deserialize_coder(coder_spec['@type'])
+  return coders.coders.deserialize_coder(coder_spec['@type'])
 
 
 class MapTask(object):
diff --git a/sdks/python/apache_beam/runners/worker/operations.py b/sdks/python/apache_beam/runners/worker/operations.py
index 5dbe57e77b49..a44561d096c8 100644
--- a/sdks/python/apache_beam/runners/worker/operations.py
+++ b/sdks/python/apache_beam/runners/worker/operations.py
@@ -281,7 +281,7 @@ def _read_side_inputs(self, tags_and_types):
 
       # Backwards compatibility for pre BEAM-733 SDKs.
       if isinstance(view_options, tuple):
-        if view_class == pvalue.SingletonPCollectionView:
+        if view_class == pvalue.AsSingleton:
           has_default, default = view_options
           view_options = {'default': default} if has_default else {}
         else:

From f32cacb7313b8352ddc054cd1b88d7a0462550db Mon Sep 17 00:00:00 2001
From: Pablo <pabloem@google.com>
Date: Mon, 22 May 2017 15:39:44 -0700
Subject: [PATCH 326/387] Adding a snippet for metrics

---
 .../examples/snippets/snippets_test.py        | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index 6654fef654cb..e302465985a4 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -32,6 +32,8 @@
 from apache_beam.coders.coders import ToStringCoder
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.examples.snippets import snippets
+from apache_beam.metrics import Metrics
+from apache_beam.metrics.metric import MetricsFilter
 from apache_beam.testing.util import assert_that
 from apache_beam.testing.util import equal_to
 from apache_beam.utils.windowed_value import WindowedValue
@@ -689,6 +691,66 @@ def test_model_co_group_by_key_tuple(self):
     expect = ['a; a@example.com; x4312', 'b; b@example.com; x8452']
     self.assertEqual(expect, self.get_output(result_path))
 
+  def test_model_use_and_query_metrics(self):
+    """DebuggingWordCount example snippets."""
+
+    import re
+
+    p = TestPipeline()  # Use TestPipeline for testing.
+    words = p | beam.Create(['albert', 'sam', 'mark', 'sarah',
+                             'swati', 'daniel', 'andrea'])
+
+    # pylint: disable=unused-variable
+    # [START metrics_usage_example]
+    class FilterTextFn(beam.DoFn):
+      """A DoFn that filters for a specific key based on a regex."""
+
+      def __init__(self, pattern):
+        self.pattern = pattern
+        # A custom metric can track values in your pipeline as it runs. Create
+        # custom metrics to count unmatched words, and know the distribution of
+        # word lengths in the input PCollection.
+        self.word_len_dist = Metrics.distribution(self.__class__,
+                                                  'word_len_dist')
+        self.unmatched_words = Metrics.counter(self.__class__,
+                                               'unmatched_words')
+
+      def process(self, element):
+        word = element
+        self.word_len_dist.update(len(word))
+        if re.match(self.pattern, word):
+          yield element
+        else:
+          self.unmatched_words.inc()
+
+    filtered_words = (
+        words | 'FilterText' >> beam.ParDo(FilterTextFn('s.*')))
+    # [END metrics_usage_example]
+    # pylint: enable=unused-variable
+
+    # [START metrics_check_values_example]
+    result = p.run()
+    result.wait_until_finish()
+
+    custom_distribution = result.metrics().query(
+        MetricsFilter().with_name('word_len_dist'))['distributions']
+    custom_counter = result.metrics().query(
+        MetricsFilter().with_name('unmatched_words'))['counters']
+
+    if custom_distribution:
+      logging.info('The average word length was %d',
+                   custom_distribution[0].committed.mean)
+    if custom_counter:
+      logging.info('There were %d words that did not match the filter.',
+                   custom_counter[0].committed)
+    # [END metrics_check_values_example]
+
+    # There should be 4 words that did not match
+    self.assertEqual(custom_counter[0].committed, 4)
+    # The shortest word is 3 characters, the longest is 6
+    self.assertEqual(custom_distribution[0].committed.min, 3)
+    self.assertEqual(custom_distribution[0].committed.max, 6)
+
   def test_model_join_using_side_inputs(self):
     name_list = ['a', 'b']
     email_list = [['a', 'a@example.com'], ['b', 'b@example.com']]

From 8b0e4bc4ca93519f014eb309b755009cc55030bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@apache.org>
Date: Wed, 24 May 2017 17:38:07 +0200
Subject: [PATCH 327/387] Update maven-dependency-plugin to version 3.0.1

---
 pom.xml               | 3 ++-
 runners/apex/pom.xml  | 5 +++++
 runners/spark/pom.xml | 4 ++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 4f833cdb8e49..7e06b5f12952 100644
--- a/pom.xml
+++ b/pom.xml
@@ -151,6 +151,7 @@
     <surefire-plugin.version>2.20</surefire-plugin.version>
     <failsafe-plugin.version>2.20</failsafe-plugin.version>
     <maven-compiler-plugin.version>3.6.1</maven-compiler-plugin.version>
+    <maven-dependency-plugin.version>3.0.1</maven-dependency-plugin.version>
     <maven-exec-plugin.version>1.4.0</maven-exec-plugin.version>
     <maven-jar-plugin.version>3.0.2</maven-jar-plugin.version>
     <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
@@ -1632,7 +1633,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven-dependency-plugin.version}</version>
         <executions>
           <execution>
             <goals><goal>analyze-only</goal></goals>
diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml
index f1a8a6245a4f..4a36bec8ab2c 100644
--- a/runners/apex/pom.xml
+++ b/runners/apex/pom.xml
@@ -78,6 +78,11 @@
     </dependency>
 
     <!-- Beam -->
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-common-runner-api</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-sdks-java-core</artifactId>
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 2c8372b754ef..697f67aadaec 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -216,6 +216,10 @@
       <version>2.10.5</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-common-runner-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-sdks-java-core</artifactId>

From 26c93681795dc9b98c3f00c2cd61c38b36656e0b Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 19 May 2017 21:35:43 -0700
Subject: [PATCH 328/387] Make SdkComponents public for
 TransformPayloadTranslator

---
 .../apache/beam/runners/core/construction/SdkComponents.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
index b0f164f4bcbd..0d3ba6093c3e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java
@@ -46,7 +46,7 @@
 import org.apache.beam.sdk.values.WindowingStrategy;
 
 /** SDK objects that will be represented at some later point within a {@link Components} object. */
-class SdkComponents {
+public class SdkComponents {
   private final RunnerApi.Components.Builder componentsBuilder;
 
   private final BiMap<AppliedPTransform<?, ?, ?>, String> transformIds;

From 1dc134cf341d1eb4fa936f6fe7b83a3edbb64687 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 19 May 2017 20:54:36 -0700
Subject: [PATCH 329/387] Centralize primitive URNs in PTransformTranslation
 class

---
 .../runners/core/construction/PTransformTranslation.java | 7 +++++++
 .../beam/runners/core/construction/ParDoTranslation.java | 9 +++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 86638dec8098..8be023ad413b 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -38,6 +38,13 @@
  * protocol buffers}.
  */
 public class PTransformTranslation {
+
+  public static final String PAR_DO_TRANSFORM_URN = "urn:beam:transform:pardo:v1";
+  public static final String FLATTEN_TRANSFORM_URN = "urn:beam:transform:flatten:v1";
+  public static final String GROUP_BY_KEY_TRANSFORM_URN = "urn:beam:transform:groupbykey:v1";
+  public static final String READ_TRANSFORM_URN = "urn:beam:transform:read:v1";
+  public static final String WINDOW_TRANSFORM_URN = "urn:beam:transform:window:v1";
+
   private static final Map<Class<? extends PTransform>, TransformPayloadTranslator>
       KNOWN_PAYLOAD_TRANSLATORS = loadTransformPayloadTranslators();
 
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
index 28d577ff2211..83277bbefd65 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -19,6 +19,7 @@
 package org.apache.beam.runners.core.construction;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.PAR_DO_TRANSFORM_URN;
 
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
@@ -74,10 +75,6 @@
  * Utilities for interacting with {@link ParDo} instances and {@link ParDoPayload} protos.
  */
 public class ParDoTranslation {
-  /**
-   * The URN for a {@link ParDoPayload}.
-   */
-  public static final String PAR_DO_PAYLOAD_URN = "urn:beam:pardo:v1";
   /**
    * The URN for an unknown Java {@link DoFn}.
    */
@@ -108,7 +105,7 @@ public FunctionSpec translate(
         AppliedPTransform<?, ?, MultiOutput<?, ?>> transform, SdkComponents components) {
       ParDoPayload payload = toProto(transform.getTransform(), components);
       return RunnerApi.FunctionSpec.newBuilder()
-          .setUrn(PAR_DO_PAYLOAD_URN)
+          .setUrn(PAR_DO_TRANSFORM_URN)
           .setParameter(Any.pack(payload))
           .build();
     }
@@ -166,7 +163,7 @@ public static TupleTag<?> getMainOutputTag(ParDoPayload payload)
   public static RunnerApi.PCollection getMainInput(
       RunnerApi.PTransform ptransform, Components components) throws IOException {
     checkArgument(
-        ptransform.getSpec().getUrn().equals(PAR_DO_PAYLOAD_URN),
+        ptransform.getSpec().getUrn().equals(PAR_DO_TRANSFORM_URN),
         "Unexpected payload type %s",
         ptransform.getSpec().getUrn());
     ParDoPayload payload = ptransform.getSpec().getParameter().unpack(ParDoPayload.class);

From 877408e078b783b3a782ae1ba0e335ea44c4a0e2 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Wed, 24 May 2017 15:28:38 -0700
Subject: [PATCH 330/387] Add URN for Splittable ProcessElement
 pseudo-primitive

---
 .../apache/beam/runners/core/construction/SplittableParDo.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
index 23ba66ab983d..dfca7d2d94eb 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
@@ -64,6 +64,9 @@ public class SplittableParDo<InputT, OutputT, RestrictionT>
     extends PTransform<PCollection<InputT>, PCollectionTuple> {
   private final ParDo.MultiOutput<InputT, OutputT> parDo;
 
+  public static final String SPLITTABLE_PROCESS_URN =
+      "urn:beam:runners_core:transforms:splittable_process:v1";
+
   /**
    * Creates the transform for the given original multi-output {@link ParDo}.
    *

From 0bf4ddbea7d0f790fad0cc8df20f0d01f38dd568 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 19 May 2017 20:55:25 -0700
Subject: [PATCH 331/387] Allow getting URN for class of transform via
 translator

---
 .../core/construction/PTransformTranslation.java   | 14 +++++++++++++-
 .../core/construction/ParDoTranslation.java        |  5 +++++
 .../core/construction/WindowIntoTranslation.java   |  7 +++++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 8be023ad413b..35bb0e3dad4e 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -116,11 +116,23 @@ private static String toProto(TupleTag<?> tag) {
     return tag.getId();
   }
 
+  public static String urnForTransform(PTransform<?, ?> transform) {
+    TransformPayloadTranslator translator =
+    KNOWN_PAYLOAD_TRANSLATORS.get(transform.getClass());
+    if (translator == null) {
+      throw new IllegalStateException(
+          String.format("No translator known for %s", transform.getClass().getName()));
+    }
+
+    return translator.getUrn(transform);
+  }
+
   /**
    * A translator consumes a {@link PTransform} application and produces the appropriate
    * FunctionSpec for a distinguished or primitive transform within the Beam runner API.
    */
   public interface TransformPayloadTranslator<T extends PTransform<?, ?>> {
-    FunctionSpec translate(AppliedPTransform<?, ?, T> transform, SdkComponents components);
+    String getUrn(T transform);
+    FunctionSpec translate(AppliedPTransform<?, ?, T> application, SdkComponents components);
   }
 }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
index 83277bbefd65..1c81f8ce05d5 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -100,6 +100,11 @@ public static TransformPayloadTranslator create() {
 
     private ParDoPayloadTranslator() {}
 
+    @Override
+    public String getUrn(ParDo.MultiOutput<?, ?> transform) {
+      return PAR_DO_TRANSFORM_URN;
+    }
+
     @Override
     public FunctionSpec translate(
         AppliedPTransform<?, ?, MultiOutput<?, ?>> transform, SdkComponents components) {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
index 215beba43cd8..33faa02f8801 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
@@ -27,6 +27,7 @@
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.WindowIntoPayload;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.Window.Assign;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
 
 /**
@@ -36,6 +37,12 @@
 public class WindowIntoTranslation {
 
   static class WindowAssignTranslator implements TransformPayloadTranslator<Window.Assign<?>> {
+
+    @Override
+    public String getUrn(Assign<?> transform) {
+      return PTransforms.WINDOW_TRANSFORM_URN;
+    }
+
     @Override
     public FunctionSpec translate(
         AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) {

From afaebb13f73868799b83c7af95cca2732e3ecb9a Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 21:00:02 -0700
Subject: [PATCH 332/387] Add registration for Read and WindowInto translators

---
 .../core/construction/ReadTranslation.java    | 74 ++++++++++++++++++-
 .../construction/WindowIntoTranslation.java   | 43 ++++++++++-
 2 files changed, 115 insertions(+), 2 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
index d6c34008f29e..aff5fc9686f7 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
@@ -20,10 +20,15 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.Any;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.BytesValue;
 import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.Map;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.IsBounded;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.ReadPayload;
@@ -32,12 +37,13 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.SerializableUtils;
 
 /**
  * Methods for translating {@link Read.Bounded} and {@link Read.Unbounded}
- * {@link PTransform PTransforms} into {@link ReadPayload} protos.
+ * {@link PTransform PTransformTranslation} into {@link ReadPayload} protos.
  */
 public class ReadTranslation {
   private static final String JAVA_SERIALIZED_BOUNDED_SOURCE = "urn:beam:java:boundedsource:v1";
@@ -124,4 +130,70 @@ private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source) {
         "BoundedSource");
   }
 
+  /**
+   * A {@link TransformPayloadTranslator} for {@link Read.Unbounded}.
+   */
+  public static class UnboundedReadPayloadTranslator
+      implements PTransformTranslation.TransformPayloadTranslator<Read.Unbounded<?>> {
+    public static TransformPayloadTranslator create() {
+      return new UnboundedReadPayloadTranslator();
+    }
+
+    private UnboundedReadPayloadTranslator() {}
+
+    @Override
+    public String getUrn(Read.Unbounded<?> transform) {
+      return PTransformTranslation.WINDOW_TRANSFORM_URN;
+    }
+
+    @Override
+    public FunctionSpec translate(
+        AppliedPTransform<?, ?, Read.Unbounded<?>> transform, SdkComponents components) {
+      ReadPayload payload = toProto(transform.getTransform());
+      return RunnerApi.FunctionSpec.newBuilder()
+          .setUrn(PTransformTranslation.READ_TRANSFORM_URN)
+          .setParameter(Any.pack(payload))
+          .build();
+    }
+  }
+
+  /**
+   * A {@link TransformPayloadTranslator} for {@link Read.Bounded}.
+   */
+  public static class BoundedReadPayloadTranslator
+      implements PTransformTranslation.TransformPayloadTranslator<Read.Bounded<?>> {
+    public static TransformPayloadTranslator create() {
+      return new BoundedReadPayloadTranslator();
+    }
+
+    private BoundedReadPayloadTranslator() {}
+
+    @Override
+    public String getUrn(Read.Bounded<?> transform) {
+      return PTransformTranslation.WINDOW_TRANSFORM_URN;
+    }
+
+    @Override
+    public FunctionSpec translate(
+        AppliedPTransform<?, ?, Read.Bounded<?>> transform, SdkComponents components) {
+      ReadPayload payload = toProto(transform.getTransform());
+      return RunnerApi.FunctionSpec.newBuilder()
+          .setUrn(PTransformTranslation.READ_TRANSFORM_URN)
+          .setParameter(Any.pack(payload))
+          .build();
+    }
+  }
+
+  /** Registers {@link UnboundedReadPayloadTranslator} and {@link BoundedReadPayloadTranslator}. */
+  @AutoService(TransformPayloadTranslatorRegistrar.class)
+  public static class Registrar implements TransformPayloadTranslatorRegistrar {
+    @Override
+    public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+        getTransformPayloadTranslators() {
+      return ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder()
+          .put(Read.Unbounded.class, new UnboundedReadPayloadTranslator())
+          .put(Read.Bounded.class, new BoundedReadPayloadTranslator())
+          .build();
+    }
+  }
 }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
index 33faa02f8801..5ed4d24906b1 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
@@ -18,14 +18,18 @@
 
 package org.apache.beam.runners.core.construction;
 
+import com.google.auto.service.AutoService;
 import com.google.protobuf.Any;
 import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.Collections;
+import java.util.Map;
 import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.WindowIntoPayload;
 import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.transforms.windowing.Window.Assign;
 import org.apache.beam.sdk.transforms.windowing.WindowFn;
@@ -40,7 +44,7 @@ static class WindowAssignTranslator implements TransformPayloadTranslator<Window
 
     @Override
     public String getUrn(Assign<?> transform) {
-      return PTransforms.WINDOW_TRANSFORM_URN;
+      return PTransformTranslation.WINDOW_TRANSFORM_URN;
     }
 
     @Override
@@ -65,4 +69,41 @@ public static WindowIntoPayload toProto(Window.Assign<?> transform, SdkComponent
     SdkFunctionSpec spec = payload.getWindowFn();
     return WindowingStrategyTranslation.windowFnFromProto(spec);
   }
+
+  /**
+   * A {@link TransformPayloadTranslator} for {@link Window}.
+   */
+  public static class WindowIntoPayloadTranslator
+      implements PTransformTranslation.TransformPayloadTranslator<Window.Assign<?>> {
+    public static TransformPayloadTranslator create() {
+      return new WindowIntoPayloadTranslator();
+    }
+
+    private WindowIntoPayloadTranslator() {}
+
+    @Override
+    public String getUrn(Window.Assign<?> transform) {
+      return PTransformTranslation.WINDOW_TRANSFORM_URN;
+    }
+
+    @Override
+    public FunctionSpec translate(
+        AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) {
+      WindowIntoPayload payload = toProto(transform.getTransform(), components);
+      return RunnerApi.FunctionSpec.newBuilder()
+          .setUrn(PTransformTranslation.WINDOW_TRANSFORM_URN)
+          .setParameter(Any.pack(payload))
+          .build();
+    }
+  }
+
+  /** Registers {@link WindowIntoPayloadTranslator}. */
+  @AutoService(TransformPayloadTranslatorRegistrar.class)
+  public static class Registrar implements TransformPayloadTranslatorRegistrar {
+    @Override
+    public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+        getTransformPayloadTranslators() {
+      return Collections.singletonMap(Window.Assign.class, new WindowIntoPayloadTranslator());
+    }
+  }
 }

From d91c840f009bdfda14f8668172d076a7e9f12f5e Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 21:09:20 -0700
Subject: [PATCH 333/387] Register ReadTranslator

---
 .../beam/runners/core/construction/ReadTranslation.java   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
index aff5fc9686f7..3ddde8d767aa 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
@@ -143,7 +143,7 @@ private UnboundedReadPayloadTranslator() {}
 
     @Override
     public String getUrn(Read.Unbounded<?> transform) {
-      return PTransformTranslation.WINDOW_TRANSFORM_URN;
+      return PTransformTranslation.READ_TRANSFORM_URN;
     }
 
     @Override
@@ -151,7 +151,7 @@ public FunctionSpec translate(
         AppliedPTransform<?, ?, Read.Unbounded<?>> transform, SdkComponents components) {
       ReadPayload payload = toProto(transform.getTransform());
       return RunnerApi.FunctionSpec.newBuilder()
-          .setUrn(PTransformTranslation.READ_TRANSFORM_URN)
+          .setUrn(getUrn(transform.getTransform()))
           .setParameter(Any.pack(payload))
           .build();
     }
@@ -170,7 +170,7 @@ private BoundedReadPayloadTranslator() {}
 
     @Override
     public String getUrn(Read.Bounded<?> transform) {
-      return PTransformTranslation.WINDOW_TRANSFORM_URN;
+      return PTransformTranslation.READ_TRANSFORM_URN;
     }
 
     @Override
@@ -178,7 +178,7 @@ public FunctionSpec translate(
         AppliedPTransform<?, ?, Read.Bounded<?>> transform, SdkComponents components) {
       ReadPayload payload = toProto(transform.getTransform());
       return RunnerApi.FunctionSpec.newBuilder()
-          .setUrn(PTransformTranslation.READ_TRANSFORM_URN)
+          .setUrn(getUrn(transform.getTransform()))
           .setParameter(Any.pack(payload))
           .build();
     }

From 69bb1be4d3bd20e5c080bcce0eb464fac3d2ff73 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 21:09:33 -0700
Subject: [PATCH 334/387] Register WindowIntoTranslator

---
 .../beam/runners/core/construction/WindowIntoTranslation.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
index 5ed4d24906b1..aa17bc90d6a3 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java
@@ -91,7 +91,7 @@ public FunctionSpec translate(
         AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) {
       WindowIntoPayload payload = toProto(transform.getTransform(), components);
       return RunnerApi.FunctionSpec.newBuilder()
-          .setUrn(PTransformTranslation.WINDOW_TRANSFORM_URN)
+          .setUrn(getUrn(transform.getTransform()))
           .setParameter(Any.pack(payload))
           .build();
     }

From 121631c62f94a6e553d3bbd8708cee2dbdf53923 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Mon, 22 May 2017 21:09:54 -0700
Subject: [PATCH 335/387] Add trivial FlattenTranslator to access URN

---
 .../core/construction/FlattenTranslator.java  | 63 +++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
new file mode 100644
index 000000000000..f1d553dc8abd
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import com.google.auto.service.AutoService;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.Window.Assign;
+
+/**
+ * Utility methods for translating a {@link Assign} to and from {@link RunnerApi} representations.
+ */
+public class FlattenTranslator implements TransformPayloadTranslator<Flatten.PCollections<?>> {
+
+  public static TransformPayloadTranslator create() {
+    return new FlattenTranslator();
+  }
+
+  private FlattenTranslator() {}
+
+  @Override
+  public String getUrn(Flatten.PCollections<?> transform) {
+    return PTransformTranslation.FLATTEN_TRANSFORM_URN;
+  }
+
+  @Override
+  public FunctionSpec translate(
+      AppliedPTransform<?, ?, Flatten.PCollections<?>> transform, SdkComponents components) {
+    return RunnerApi.FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform())).build();
+  }
+
+  /** Registers {@link FlattenTranslator}. */
+  @AutoService(TransformPayloadTranslatorRegistrar.class)
+  public static class Registrar implements TransformPayloadTranslatorRegistrar {
+    @Override
+    public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+        getTransformPayloadTranslators() {
+      return Collections.singletonMap(Flatten.PCollections.class, new FlattenTranslator());
+    }
+  }
+}

From 8e09596a981fe69edb7a1560e864bc852978ba81 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Tue, 23 May 2017 21:15:48 -0700
Subject: [PATCH 336/387] Add transform-analysis helpers to ReadTranslation

These helpers allow a runner to extract a source and inspect boundedness
without reference to the specific user-facing Java classes Read.Bounded
and Read.Unbounded.
---
 .../construction/PCollectionTranslation.java  |  4 +-
 .../core/construction/ReadTranslation.java    | 43 +++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
index 303c02d44a11..968966f459e1 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionTranslation.java
@@ -64,7 +64,7 @@ public static Coder<?> getCoder(
         components.getWindowingStrategiesOrThrow(pCollection.getWindowingStrategyId()), components);
   }
 
-  private static RunnerApi.IsBounded toProto(IsBounded bounded) {
+  static RunnerApi.IsBounded toProto(IsBounded bounded) {
     switch (bounded) {
       case BOUNDED:
         return RunnerApi.IsBounded.BOUNDED;
@@ -76,7 +76,7 @@ private static RunnerApi.IsBounded toProto(IsBounded bounded) {
     }
   }
 
-  private static IsBounded fromProto(RunnerApi.IsBounded isBounded) {
+  static IsBounded fromProto(RunnerApi.IsBounded isBounded) {
     switch (isBounded) {
       case BOUNDED:
         return IsBounded.BOUNDED;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
index 3ddde8d767aa..572384bdd549 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java
@@ -26,6 +26,8 @@
 import com.google.protobuf.ByteString;
 import com.google.protobuf.BytesValue;
 import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
@@ -40,6 +42,8 @@
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
 
 /**
  * Methods for translating {@link Read.Bounded} and {@link Read.Unbounded}
@@ -102,6 +106,29 @@ public static BoundedSource<?> boundedSourceFromProto(ReadPayload payload)
         "BoundedSource");
   }
 
+  public static <T> BoundedSource<T> boundedSourceFromTransform(
+      AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform)
+      throws IOException {
+    return (BoundedSource<T>) boundedSourceFromProto(getReadPayload(transform));
+  }
+
+  public static <T, CheckpointT extends UnboundedSource.CheckpointMark>
+      UnboundedSource<T, CheckpointT> unboundedSourceFromTransform(
+          AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform)
+          throws IOException {
+    return (UnboundedSource<T, CheckpointT>) unboundedSourceFromProto(getReadPayload(transform));
+  }
+
+  private static <T> ReadPayload getReadPayload(
+      AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform)
+      throws IOException {
+    return PTransformTranslation.toProto(
+            transform, Collections.<AppliedPTransform<?, ?, ?>>emptyList(), SdkComponents.create())
+        .getSpec()
+        .getParameter()
+        .unpack(ReadPayload.class);
+  }
+
   private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source) {
     return SdkFunctionSpec.newBuilder()
         .setSpec(
@@ -130,6 +157,22 @@ private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source) {
         "BoundedSource");
   }
 
+  public static PCollection.IsBounded sourceIsBounded(AppliedPTransform<?, ?, ?> transform) {
+    try {
+      return PCollectionTranslation.fromProto(
+          PTransformTranslation.toProto(
+                  transform,
+                  Collections.<AppliedPTransform<?, ?, ?>>emptyList(),
+                  SdkComponents.create())
+              .getSpec()
+              .getParameter()
+              .unpack(ReadPayload.class)
+              .getIsBounded());
+    } catch (IOException e) {
+      throw new RuntimeException("Internal error determining boundedness of Read", e);
+    }
+  }
+
   /**
    * A {@link TransformPayloadTranslator} for {@link Read.Unbounded}.
    */

From 663ad88178ceadffe4cfa592555986ed7dde58b4 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 19 May 2017 21:28:19 -0700
Subject: [PATCH 337/387] Add RawPTransform, which can just vend its URN and
 payload

This is the type that will be returned when a pipeline is deserialized.
This also is convenient for direct runner overrides which do not really
merit translator registrations, yet URNs need to be known in order to
key the evaluator registry off URN.
---
 .../construction/PTransformTranslation.java   | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 35bb0e3dad4e..9f5f3b50b723 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -21,15 +21,20 @@
 import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.protobuf.Any;
+import com.google.protobuf.Message;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.ServiceLoader;
+import javax.annotation.Nullable;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.TupleTag;
 
@@ -135,4 +140,50 @@ public interface TransformPayloadTranslator<T extends PTransform<?, ?>> {
     String getUrn(T transform);
     FunctionSpec translate(AppliedPTransform<?, ?, T> application, SdkComponents components);
   }
+
+  /**
+   * A {@link PTransform} that indicates its URN and payload directly.
+   *
+   * <p>This is the result of rehydrating transforms from a pipeline proto. There is no {@link
+   * #expand} method since the definition of the transform may be lost. The transform is already
+   * fully expanded in the pipeline proto.
+   */
+  public abstract static class RawPTransform<
+          InputT extends PInput, OutputT extends POutput, PayloadT extends Message>
+      extends PTransform<InputT, OutputT> {
+
+    public abstract String getUrn();
+
+    @Nullable
+    PayloadT getPayload() {
+      return null;
+    }
+  }
+
+  /**
+   * A translator that uses the explicit URN and payload from a {@link RawPTransform}.
+   */
+  public static class RawPTransformTranslator<PayloadT extends Message>
+      implements TransformPayloadTranslator<RawPTransform<?, ?, PayloadT>> {
+    @Override
+    public String getUrn(RawPTransform<?, ?, PayloadT> transform) {
+      return transform.getUrn();
+    }
+
+    @Override
+    public FunctionSpec translate(
+        AppliedPTransform<?, ?, RawPTransform<?, ?, PayloadT>> transform,
+        SdkComponents components) {
+      PayloadT payload = transform.getTransform().getPayload();
+
+      FunctionSpec.Builder transformSpec =
+          FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform()));
+
+      if (payload != null) {
+        transformSpec.setParameter(Any.pack(payload));
+      }
+
+      return transformSpec.build();
+    }
+  }
 }

From 0e29cc52a3e4b0d9ae5ff3907f10e4e87b734186 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Fri, 19 May 2017 20:53:32 -0700
Subject: [PATCH 338/387] URNs for DirectRunner TransformEvaluator and
 RootInputProvider

This makes two of the Java DirectRunner's registries key off URN instead
of Java class. A root TransformEvaluator requires shards generated by
its associated RootInputProvider, hence changing both at once.
---
 runners/direct-java/pom.xml                   |   5 +
 .../direct/BoundedReadEvaluatorFactory.java   |  14 +-
 .../beam/runners/direct/DirectGroupByKey.java |  21 ++-
 .../runners/direct/EmptyInputProvider.java    |   8 +-
 .../direct/ParDoMultiOverrideFactory.java     |  13 +-
 .../runners/direct/ReadEvaluatorFactory.java  |  97 +++++++++++++
 .../runners/direct/RootInputProvider.java     |   7 +-
 .../runners/direct/RootProviderRegistry.java  |  28 ++--
 .../beam/runners/direct/SourceShard.java      |  33 +++++
 .../direct/TestStreamEvaluatorFactory.java    |  28 ++--
 .../direct/TransformEvaluatorRegistry.java    | 128 +++++++++++++-----
 .../direct/UnboundedReadEvaluatorFactory.java |  31 +++--
 .../runners/direct/ViewOverrideFactory.java   |  12 +-
 .../main/resources/beam/findbugs-filter.xml   |   7 +
 14 files changed, 344 insertions(+), 88 deletions(-)
 create mode 100644 runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java
 create mode 100644 runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java

diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index bec21139d989..cba4b099ff0f 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -207,6 +207,11 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
index 76db861897e0..fcaaa8442082 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
@@ -33,10 +33,10 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.ReadTranslation;
 import org.apache.beam.runners.direct.StepTransformResult.Builder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.BoundedSource.BoundedReader;
-import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.Read.Bounded;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -180,16 +180,17 @@ public TransformResult<BoundedSourceShard<OutputT>> finishBundle()  {
   }
 
   @AutoValue
-  abstract static class BoundedSourceShard<T> {
+  abstract static class BoundedSourceShard<T> implements SourceShard<T> {
     static <T> BoundedSourceShard<T> of(BoundedSource<T> source) {
       return new AutoValue_BoundedReadEvaluatorFactory_BoundedSourceShard<>(source);
     }
 
-    abstract BoundedSource<T> getSource();
+    @Override
+    public abstract BoundedSource<T> getSource();
   }
 
   static class InputProvider<T>
-      implements RootInputProvider<T, BoundedSourceShard<T>, PBegin, Read.Bounded<T>> {
+      implements RootInputProvider<T, BoundedSourceShard<T>, PBegin> {
     private final EvaluationContext evaluationContext;
 
     InputProvider(EvaluationContext evaluationContext) {
@@ -198,9 +199,10 @@ static class InputProvider<T>
 
     @Override
     public Collection<CommittedBundle<BoundedSourceShard<T>>> getInitialInputs(
-        AppliedPTransform<PBegin, PCollection<T>, Read.Bounded<T>> transform, int targetParallelism)
+        AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform,
+        int targetParallelism)
         throws Exception {
-      BoundedSource<T> source = transform.getTransform().getSource();
+      BoundedSource<T> source = ReadTranslation.boundedSourceFromTransform(transform);
       PipelineOptions options = evaluationContext.getPipelineOptions();
       long estimatedBytes = source.getEstimatedSizeBytes(options);
       long bytesPerBundle = estimatedBytes / targetParallelism;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
index 791615a2a194..f239070d925b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
@@ -20,9 +20,11 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import com.google.protobuf.Message;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItemCoder;
 import org.apache.beam.runners.core.construction.ForwardingPTransform;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.IterableCoder;
@@ -37,6 +39,9 @@ class DirectGroupByKey<K, V>
     extends ForwardingPTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> {
   private final GroupByKey<K, V> original;
 
+  static final String DIRECT_GBKO_URN = "urn:beam:directrunner:transforms:gbko:v1";
+  static final String DIRECT_GABW_URN = "urn:beam:directrunner:transforms:gabw:v1";
+
   DirectGroupByKey(GroupByKey<K, V> from) {
     this.original = from;
   }
@@ -68,7 +73,8 @@ public PCollection<KV<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
   }
 
   static final class DirectGroupByKeyOnly<K, V>
-      extends PTransform<PCollection<KV<K, V>>, PCollection<KeyedWorkItem<K, V>>> {
+      extends PTransformTranslation.RawPTransform<
+          PCollection<KV<K, V>>, PCollection<KeyedWorkItem<K, V>>, Message> {
     @Override
     public PCollection<KeyedWorkItem<K, V>> expand(PCollection<KV<K, V>> input) {
       return PCollection.createPrimitiveOutputInternal(
@@ -86,10 +92,16 @@ protected Coder<?> getDefaultOutputCoder(
           GroupByKey.getInputValueCoder(input.getCoder()),
           input.getWindowingStrategy().getWindowFn().windowCoder());
     }
+
+    @Override
+    public String getUrn() {
+      return DIRECT_GBKO_URN;
+    }
   }
 
   static final class DirectGroupAlsoByWindow<K, V>
-      extends PTransform<PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>> {
+      extends PTransformTranslation.RawPTransform<
+          PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>, Message> {
 
     private final WindowingStrategy<?, ?> inputWindowingStrategy;
     private final WindowingStrategy<?, ?> outputWindowingStrategy;
@@ -135,5 +147,10 @@ public PCollection<KV<K, Iterable<V>>> expand(PCollection<KeyedWorkItem<K, V>> i
       return PCollection.createPrimitiveOutputInternal(
           input.getPipeline(), outputWindowingStrategy, input.isBounded());
     }
+
+    @Override
+    public String getUrn() {
+      return DIRECT_GABW_URN;
+    }
   }
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
index c36879a5f7b6..a5a53bc8ea6c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyInputProvider.java
@@ -20,13 +20,12 @@
 import java.util.Collection;
 import java.util.Collections;
 import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
 
 /** A {@link RootInputProvider} that provides a singleton empty bundle. */
-class EmptyInputProvider<T>
-    implements RootInputProvider<T, Void, PCollectionList<T>, Flatten.PCollections<T>> {
+class EmptyInputProvider<T> implements RootInputProvider<T, Void, PCollectionList<T>> {
   EmptyInputProvider() {}
 
   /**
@@ -36,7 +35,8 @@ class EmptyInputProvider<T>
    */
   @Override
   public Collection<CommittedBundle<Void>> getInitialInputs(
-      AppliedPTransform<PCollectionList<T>, PCollection<T>, Flatten.PCollections<T>>
+      AppliedPTransform<
+              PCollectionList<T>, PCollection<T>, PTransform<PCollectionList<T>, PCollection<T>>>
           transform,
       int targetParallelism) {
     return Collections.emptyList();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
index be433db1ca00..df2054b333ab 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java
@@ -19,11 +19,13 @@
 
 import static com.google.common.base.Preconditions.checkState;
 
+import com.google.protobuf.Message;
 import java.util.Map;
 import org.apache.beam.runners.core.KeyedWorkItem;
 import org.apache.beam.runners.core.KeyedWorkItemCoder;
 import org.apache.beam.runners.core.KeyedWorkItems;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
 import org.apache.beam.runners.core.construction.ReplacementOutputs;
 import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
@@ -165,8 +167,12 @@ public PCollectionTuple expand(PCollection<KV<K, InputT>> input) {
     }
   }
 
+  static final String DIRECT_STATEFUL_PAR_DO_URN =
+      "urn:beam:directrunner:transforms:stateful_pardo:v1";
+
   static class StatefulParDo<K, InputT, OutputT>
-      extends PTransform<PCollection<? extends KeyedWorkItem<K, KV<K, InputT>>>, PCollectionTuple> {
+      extends PTransformTranslation.RawPTransform<
+          PCollection<? extends KeyedWorkItem<K, KV<K, InputT>>>, PCollectionTuple, Message> {
     private final transient MultiOutput<KV<K, InputT>, OutputT> underlyingParDo;
     private final transient PCollection<KV<K, InputT>> originalInput;
 
@@ -201,6 +207,11 @@ public PCollectionTuple expand(PCollection<? extends KeyedWorkItem<K, KV<K, Inpu
 
       return outputs;
     }
+
+    @Override
+    public String getUrn() {
+      return DIRECT_STATEFUL_PAR_DO_URN;
+    }
   }
 
   /**
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java
new file mode 100644
index 000000000000..852170661773
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ReadEvaluatorFactory.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import java.util.Collection;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.ReadTranslation;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * A {@link TransformEvaluatorFactory} that produces {@link TransformEvaluator TransformEvaluators}
+ * for the {@link Read Read} primitives, whether bounded or unbounded.
+ */
+final class ReadEvaluatorFactory implements TransformEvaluatorFactory {
+
+  final BoundedReadEvaluatorFactory boundedFactory;
+  final UnboundedReadEvaluatorFactory unboundedFactory;
+
+  public ReadEvaluatorFactory(EvaluationContext context) {
+    boundedFactory = new BoundedReadEvaluatorFactory(context);
+    unboundedFactory = new UnboundedReadEvaluatorFactory(context);
+  }
+
+  @Nullable
+  @Override
+  public <InputT> TransformEvaluator<InputT> forApplication(
+      AppliedPTransform<?, ?, ?> application, CommittedBundle<?> inputBundle) throws Exception {
+    switch (ReadTranslation.sourceIsBounded(application)) {
+      case BOUNDED:
+        return boundedFactory.forApplication(application, inputBundle);
+      case UNBOUNDED:
+        return unboundedFactory.forApplication(application, inputBundle);
+      default:
+        throw new IllegalArgumentException("PCollection is neither bounded nor unbounded?!?");
+    }
+  }
+
+  @Override
+  public void cleanup() throws Exception {
+    boundedFactory.cleanup();
+    unboundedFactory.cleanup();
+  }
+
+  static <T> InputProvider<T> inputProvider(EvaluationContext context) {
+    return new InputProvider(context);
+  }
+
+  private static class InputProvider<T> implements RootInputProvider<T, SourceShard<T>, PBegin> {
+
+    private final UnboundedReadEvaluatorFactory.InputProvider<T> unboundedInputProvider;
+    private final BoundedReadEvaluatorFactory.InputProvider<T> boundedInputProvider;
+
+    InputProvider(EvaluationContext context) {
+      this.unboundedInputProvider = new UnboundedReadEvaluatorFactory.InputProvider<T>(context);
+      this.boundedInputProvider = new BoundedReadEvaluatorFactory.InputProvider<T>(context);
+    }
+
+    @Override
+    public Collection<CommittedBundle<SourceShard<T>>> getInitialInputs(
+        AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>>
+            appliedTransform,
+        int targetParallelism)
+        throws Exception {
+      switch (ReadTranslation.sourceIsBounded(appliedTransform)) {
+        case BOUNDED:
+          // This cast could be made unnecessary, but too much bounded polymorphism
+          return (Collection)
+              boundedInputProvider.getInitialInputs(appliedTransform, targetParallelism);
+        case UNBOUNDED:
+          // This cast could be made unnecessary, but too much bounded polymorphism
+          return (Collection)
+              unboundedInputProvider.getInitialInputs(appliedTransform, targetParallelism);
+        default:
+          throw new IllegalArgumentException("PCollection is neither bounded nor unbounded?!?");
+      }
+    }
+  }
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
index ce6951805117..0b3de3226aca 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootInputProvider.java
@@ -29,8 +29,7 @@
  * Provides {@link CommittedBundle bundles} that will be provided to the {@link PTransform
  * PTransforms} that are at the root of a {@link Pipeline}.
  */
-interface RootInputProvider<
-    T, ShardT, InputT extends PInput, TransformT extends PTransform<InputT, PCollection<T>>> {
+interface RootInputProvider<T, ShardT, InputT extends PInput> {
   /**
    * Get the initial inputs for the {@link AppliedPTransform}. The {@link AppliedPTransform} will be
    * provided with these {@link CommittedBundle bundles} as input when the {@link Pipeline} runs.
@@ -44,6 +43,8 @@ interface RootInputProvider<
    *     greater than or equal to 1.
    */
   Collection<CommittedBundle<ShardT>> getInitialInputs(
-      AppliedPTransform<InputT, PCollection<T>, TransformT> transform, int targetParallelism)
+      AppliedPTransform<InputT, PCollection<T>, PTransform<InputT, PCollection<T>>>
+          transform,
+      int targetParallelism)
       throws Exception;
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
index 4b0c06d3611f..5cbeab7db249 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java
@@ -18,13 +18,14 @@
 package org.apache.beam.runners.direct;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.FLATTEN_TRANSFORM_URN;
+import static org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DIRECT_TEST_STREAM_URN;
 
 import com.google.common.collect.ImmutableMap;
 import java.util.Collection;
 import java.util.Map;
-import org.apache.beam.sdk.io.Read;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
 import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Flatten.PCollections;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
@@ -33,34 +34,31 @@
  */
 class RootProviderRegistry {
   public static RootProviderRegistry defaultRegistry(EvaluationContext context) {
-    ImmutableMap.Builder<Class<? extends PTransform>, RootInputProvider<?, ?, ?, ?>>
+    ImmutableMap.Builder<String, RootInputProvider<?, ?, ?>>
         defaultProviders = ImmutableMap.builder();
     defaultProviders
-        .put(Read.Bounded.class, new BoundedReadEvaluatorFactory.InputProvider(context))
-        .put(Read.Unbounded.class, new UnboundedReadEvaluatorFactory.InputProvider(context))
-        .put(
-            TestStreamEvaluatorFactory.DirectTestStreamFactory.DirectTestStream.class,
-            new TestStreamEvaluatorFactory.InputProvider(context))
-        .put(PCollections.class, new EmptyInputProvider());
+        .put(PTransformTranslation.READ_TRANSFORM_URN, ReadEvaluatorFactory.inputProvider(context))
+        .put(DIRECT_TEST_STREAM_URN, new TestStreamEvaluatorFactory.InputProvider(context))
+        .put(FLATTEN_TRANSFORM_URN, new EmptyInputProvider());
     return new RootProviderRegistry(defaultProviders.build());
   }
 
-  private final Map<Class<? extends PTransform>, RootInputProvider<?, ?, ?, ?>> providers;
+  private final Map<String, RootInputProvider<?, ?, ?>> providers;
 
   private RootProviderRegistry(
-      Map<Class<? extends PTransform>, RootInputProvider<?, ?, ?, ?>> providers) {
+      Map<String, RootInputProvider<?, ?, ?>> providers) {
     this.providers = providers;
   }
 
   public Collection<CommittedBundle<?>> getInitialInputs(
       AppliedPTransform<?, ?, ?> transform, int targetParallelism) throws Exception {
-    Class<? extends PTransform> transformClass = transform.getTransform().getClass();
+    String transformUrn = PTransformTranslation.urnForTransform(transform.getTransform());
     RootInputProvider provider =
         checkNotNull(
-            providers.get(transformClass),
-            "Tried to get a %s for a Transform of type %s, but there is no such provider",
+            providers.get(transformUrn),
+            "Tried to get a %s for a transform \"%s\", but there is no such provider",
             RootInputProvider.class.getSimpleName(),
-            transformClass.getSimpleName());
+            transformUrn);
     return provider.getInitialInputs(transform, targetParallelism);
   }
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java
new file mode 100644
index 000000000000..a054333d70fc
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SourceShard.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.io.UnboundedSource;
+
+/**
+ * A shard for a source in the {@link Read} transform.
+ *
+ * <p>Since {@link UnboundedSource} and {@link BoundedSource} have radically different needs, this
+ * is a mostly-empty interface.
+ */
+interface SourceShard<T> {
+  Source<T> getSource();
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
index 8b21d5affd60..b1db58f7c4a0 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
@@ -22,12 +22,14 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
 import com.google.common.collect.Iterables;
+import com.google.protobuf.Message;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
 import org.apache.beam.runners.core.construction.ReplacementOutputs;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
@@ -180,7 +182,10 @@ public Map<PValue, ReplacementOutput> mapOutputs(
       return ReplacementOutputs.singleton(outputs, newOutput);
     }
 
-    static class DirectTestStream<T> extends PTransform<PBegin, PCollection<T>> {
+    static final String DIRECT_TEST_STREAM_URN = "urn:beam:directrunner:transforms:test_stream:v1";
+
+    static class DirectTestStream<T>
+        extends PTransformTranslation.RawPTransform<PBegin, PCollection<T>, Message> {
       private final transient DirectRunner runner;
       private final TestStream<T> original;
 
@@ -197,12 +202,15 @@ public PCollection<T> expand(PBegin input) {
                 input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
             .setCoder(original.getValueCoder());
       }
+
+      @Override
+      public String getUrn() {
+        return DIRECT_TEST_STREAM_URN;
+      }
     }
   }
 
-  static class InputProvider<T>
-      implements RootInputProvider<
-          T, TestStreamIndex<T>, PBegin, DirectTestStreamFactory.DirectTestStream<T>> {
+  static class InputProvider<T> implements RootInputProvider<T, TestStreamIndex<T>, PBegin> {
     private final EvaluationContext evaluationContext;
 
     InputProvider(EvaluationContext evaluationContext) {
@@ -211,15 +219,17 @@ static class InputProvider<T>
 
     @Override
     public Collection<CommittedBundle<TestStreamIndex<T>>> getInitialInputs(
-        AppliedPTransform<PBegin, PCollection<T>, DirectTestStreamFactory.DirectTestStream<T>>
-            transform,
+        AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform,
         int targetParallelism) {
+
+      // This will always be run on an execution-time transform, so it can be downcast
+      DirectTestStreamFactory.DirectTestStream<T> testStream =
+          (DirectTestStreamFactory.DirectTestStream<T>) transform.getTransform();
+
       CommittedBundle<TestStreamIndex<T>> initialBundle =
           evaluationContext
               .<TestStreamIndex<T>>createRootBundle()
-              .add(
-                  WindowedValue.valueInGlobalWindow(
-                      TestStreamIndex.of(transform.getTransform().original)))
+              .add(WindowedValue.valueInGlobalWindow(TestStreamIndex.of(testStream.original)))
               .commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
       return Collections.singleton(initialBundle);
     }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
index 718cca2f5ddb..d144b2042d2b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
@@ -19,23 +19,33 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.FLATTEN_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.PAR_DO_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.READ_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.WINDOW_TRANSFORM_URN;
+import static org.apache.beam.runners.core.construction.SplittableParDo.SPLITTABLE_PROCESS_URN;
+import static org.apache.beam.runners.direct.DirectGroupByKey.DIRECT_GABW_URN;
+import static org.apache.beam.runners.direct.DirectGroupByKey.DIRECT_GBKO_URN;
+import static org.apache.beam.runners.direct.ParDoMultiOverrideFactory.DIRECT_STATEFUL_PAR_DO_URN;
+import static org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DIRECT_TEST_STREAM_URN;
+import static org.apache.beam.runners.direct.ViewOverrideFactory.DIRECT_WRITE_VIEW_URN;
 
+import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableMap;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
-import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
-import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
-import org.apache.beam.runners.direct.ParDoMultiOverrideFactory.StatefulParDo;
-import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
-import org.apache.beam.sdk.io.Read;
+import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessElements;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.runners.core.construction.SdkComponents;
+import org.apache.beam.runners.core.construction.TransformPayloadTranslatorRegistrar;
+import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DirectTestStream;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Flatten.PCollections;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.windowing.Window;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,43 +55,93 @@
  */
 class TransformEvaluatorRegistry implements TransformEvaluatorFactory {
   private static final Logger LOG = LoggerFactory.getLogger(TransformEvaluatorRegistry.class);
+
   public static TransformEvaluatorRegistry defaultRegistry(EvaluationContext ctxt) {
-    @SuppressWarnings({"rawtypes"})
-    ImmutableMap<Class<? extends PTransform>, TransformEvaluatorFactory> primitives =
-        ImmutableMap.<Class<? extends PTransform>, TransformEvaluatorFactory>builder()
-            .put(Read.Bounded.class, new BoundedReadEvaluatorFactory(ctxt))
-            .put(Read.Unbounded.class, new UnboundedReadEvaluatorFactory(ctxt))
+    ImmutableMap<String, TransformEvaluatorFactory> primitives =
+        ImmutableMap.<String, TransformEvaluatorFactory>builder()
+            // Beam primitives
+            .put(READ_TRANSFORM_URN, new ReadEvaluatorFactory(ctxt))
             .put(
-                ParDo.MultiOutput.class,
+                PAR_DO_TRANSFORM_URN,
                 new ParDoEvaluatorFactory<>(ctxt, ParDoEvaluator.defaultRunnerFactory()))
-            .put(StatefulParDo.class, new StatefulParDoEvaluatorFactory<>(ctxt))
-            .put(PCollections.class, new FlattenEvaluatorFactory(ctxt))
-            .put(WriteView.class, new ViewEvaluatorFactory(ctxt))
-            .put(Window.Assign.class, new WindowEvaluatorFactory(ctxt))
-            // Runner-specific primitives used in expansion of GroupByKey
-            .put(DirectGroupByKeyOnly.class, new GroupByKeyOnlyEvaluatorFactory(ctxt))
-            .put(DirectGroupAlsoByWindow.class, new GroupAlsoByWindowEvaluatorFactory(ctxt))
-            .put(
-                TestStreamEvaluatorFactory.DirectTestStreamFactory.DirectTestStream.class,
-                new TestStreamEvaluatorFactory(ctxt))
-            // Runner-specific primitive used in expansion of SplittableParDo
-            .put(
-                SplittableParDoViaKeyedWorkItems.ProcessElements.class,
-                new SplittableProcessElementsEvaluatorFactory<>(ctxt))
+            .put(FLATTEN_TRANSFORM_URN, new FlattenEvaluatorFactory(ctxt))
+            .put(WINDOW_TRANSFORM_URN, new WindowEvaluatorFactory(ctxt))
+
+            // Runner-specific primitives
+            .put(DIRECT_WRITE_VIEW_URN, new ViewEvaluatorFactory(ctxt))
+            .put(DIRECT_STATEFUL_PAR_DO_URN, new StatefulParDoEvaluatorFactory<>(ctxt))
+            .put(DIRECT_GBKO_URN, new GroupByKeyOnlyEvaluatorFactory(ctxt))
+            .put(DIRECT_GABW_URN, new GroupAlsoByWindowEvaluatorFactory(ctxt))
+            .put(DIRECT_TEST_STREAM_URN, new TestStreamEvaluatorFactory(ctxt))
+
+            // Runners-core primitives
+            .put(SPLITTABLE_PROCESS_URN, new SplittableProcessElementsEvaluatorFactory<>(ctxt))
             .build();
     return new TransformEvaluatorRegistry(primitives);
   }
 
+  /** Registers classes specialized to the direct runner. */
+  @AutoService(TransformPayloadTranslatorRegistrar.class)
+  public static class DirectTransformsRegistrar implements TransformPayloadTranslatorRegistrar {
+    @Override
+    public Map<
+            ? extends Class<? extends PTransform>,
+            ? extends PTransformTranslation.TransformPayloadTranslator>
+        getTransformPayloadTranslators() {
+      return ImmutableMap
+          .<Class<? extends PTransform>, PTransformTranslation.TransformPayloadTranslator>builder()
+          .put(
+              DirectGroupByKey.DirectGroupByKeyOnly.class,
+              new PTransformTranslation.RawPTransformTranslator<>())
+          .put(
+              DirectGroupByKey.DirectGroupAlsoByWindow.class,
+              new PTransformTranslation.RawPTransformTranslator())
+          .put(
+              ParDoMultiOverrideFactory.StatefulParDo.class,
+              new PTransformTranslation.RawPTransformTranslator<>())
+          .put(
+              ViewOverrideFactory.WriteView.class,
+              new PTransformTranslation.RawPTransformTranslator<>())
+          .put(DirectTestStream.class, new PTransformTranslation.RawPTransformTranslator<>())
+          .put(
+              SplittableParDoViaKeyedWorkItems.ProcessElements.class,
+              new SplittableParDoProcessElementsTranslator())
+          .build();
+    }
+  }
+
+  /**
+   * A translator just to vend the URN. This will need to be moved to runners-core-construction-java
+   * once SDF is reorganized appropriately.
+   */
+  private static class SplittableParDoProcessElementsTranslator
+      implements TransformPayloadTranslator<ProcessElements<?, ?, ?, ?>> {
+
+    private SplittableParDoProcessElementsTranslator() {}
+
+    @Override
+    public String getUrn(ProcessElements<?, ?, ?, ?> transform) {
+      return SPLITTABLE_PROCESS_URN;
+    }
+
+    @Override
+    public FunctionSpec translate(
+        AppliedPTransform<?, ?, ProcessElements<?, ?, ?, ?>> transform, SdkComponents components) {
+      throw new UnsupportedOperationException(
+          String.format("%s should never be translated",
+          ProcessElements.class.getCanonicalName()));
+    }
+  }
+
   // the TransformEvaluatorFactories can construct instances of all generic types of transform,
   // so all instances of a primitive can be handled with the same evaluator factory.
-  @SuppressWarnings("rawtypes")
-  private final Map<Class<? extends PTransform>, TransformEvaluatorFactory> factories;
+  private final Map<String, TransformEvaluatorFactory> factories;
 
   private final AtomicBoolean finished = new AtomicBoolean(false);
 
   private TransformEvaluatorRegistry(
       @SuppressWarnings("rawtypes")
-      Map<Class<? extends PTransform>, TransformEvaluatorFactory> factories) {
+      Map<String, TransformEvaluatorFactory> factories) {
     this.factories = factories;
   }
 
@@ -91,10 +151,12 @@ public <InputT> TransformEvaluator<InputT> forApplication(
       throws Exception {
     checkState(
         !finished.get(), "Tried to get an evaluator for a finished TransformEvaluatorRegistry");
-    Class<? extends PTransform> transformClass = application.getTransform().getClass();
+
+    String urn = PTransformTranslation.urnForTransform(application.getTransform());
+
     TransformEvaluatorFactory factory =
         checkNotNull(
-            factories.get(transformClass), "No evaluator for PTransform type %s", transformClass);
+            factories.get(urn), "No evaluator for PTransform \"%s\"", urn);
     return factory.forApplication(application, inputBundle);
   }
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
index cba826ccc98c..7d4bba112385 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.concurrent.ThreadLocalRandom;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.ReadTranslation;
 import org.apache.beam.runners.direct.UnboundedReadDeduplicator.NeverDeduplicator;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.Read.Unbounded;
@@ -253,7 +254,8 @@ public TransformResult<UnboundedSourceShard<OutputT, CheckpointMarkT>> finishBun
   }
 
   @AutoValue
-  abstract static class UnboundedSourceShard<T, CheckpointT extends CheckpointMark> {
+  abstract static class UnboundedSourceShard<T, CheckpointT extends CheckpointMark>
+      implements SourceShard<T> {
     static <T, CheckpointT extends CheckpointMark> UnboundedSourceShard<T, CheckpointT> unstarted(
         UnboundedSource<T, CheckpointT> source, UnboundedReadDeduplicator deduplicator) {
       return of(source, deduplicator, null, null);
@@ -268,7 +270,8 @@ static <T, CheckpointT extends CheckpointMark> UnboundedSourceShard<T, Checkpoin
           source, deduplicator, reader, checkpoint);
     }
 
-    abstract UnboundedSource<T, CheckpointT> getSource();
+    @Override
+    public abstract UnboundedSource<T, CheckpointT> getSource();
 
     abstract UnboundedReadDeduplicator getDeduplicator();
 
@@ -283,9 +286,8 @@ UnboundedSourceShard<T, CheckpointT> withCheckpoint(CheckpointT newCheckpoint) {
     }
   }
 
-  static class InputProvider<OutputT>
-      implements RootInputProvider<
-          OutputT, UnboundedSourceShard<OutputT, ?>, PBegin, Unbounded<OutputT>> {
+  static class InputProvider<T>
+      implements RootInputProvider<T, UnboundedSourceShard<T, ?>, PBegin> {
     private final EvaluationContext evaluationContext;
 
     InputProvider(EvaluationContext evaluationContext) {
@@ -293,27 +295,28 @@ static class InputProvider<OutputT>
     }
 
     @Override
-    public Collection<CommittedBundle<UnboundedSourceShard<OutputT, ?>>> getInitialInputs(
-        AppliedPTransform<PBegin, PCollection<OutputT>, Unbounded<OutputT>> transform,
+    public Collection<CommittedBundle<UnboundedSourceShard<T, ?>>> getInitialInputs(
+        AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>>
+            transform,
         int targetParallelism)
         throws Exception {
-      UnboundedSource<OutputT, ?> source = transform.getTransform().getSource();
-      List<? extends UnboundedSource<OutputT, ?>> splits =
+      UnboundedSource<T, ?> source = ReadTranslation.unboundedSourceFromTransform(transform);
+      List<? extends UnboundedSource<T, ?>> splits =
           source.split(targetParallelism, evaluationContext.getPipelineOptions());
       UnboundedReadDeduplicator deduplicator =
           source.requiresDeduping()
               ? UnboundedReadDeduplicator.CachedIdDeduplicator.create()
               : NeverDeduplicator.create();
 
-      ImmutableList.Builder<CommittedBundle<UnboundedSourceShard<OutputT, ?>>> initialShards =
+      ImmutableList.Builder<CommittedBundle<UnboundedSourceShard<T, ?>>> initialShards =
           ImmutableList.builder();
-      for (UnboundedSource<OutputT, ?> split : splits) {
-        UnboundedSourceShard<OutputT, ?> shard =
+      for (UnboundedSource<T, ?> split : splits) {
+        UnboundedSourceShard<T, ?> shard =
             UnboundedSourceShard.unstarted(split, deduplicator);
         initialShards.add(
             evaluationContext
-                .<UnboundedSourceShard<OutputT, ?>>createRootBundle()
-                .add(WindowedValue.<UnboundedSourceShard<OutputT, ?>>valueInGlobalWindow(shard))
+                .<UnboundedSourceShard<T, ?>>createRootBundle()
+                .add(WindowedValue.<UnboundedSourceShard<T, ?>>valueInGlobalWindow(shard))
                 .commit(BoundedWindow.TIMESTAMP_MAX_VALUE));
       }
       return initialShards.build();
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
index b3bbac827fd3..501b4365835a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
@@ -18,10 +18,12 @@
 
 package org.apache.beam.runners.direct;
 
+import com.google.protobuf.Message;
 import java.util.Collections;
 import java.util.Map;
 import org.apache.beam.runners.core.construction.ForwardingPTransform;
 import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -93,7 +95,7 @@ protected PTransform<PCollection<ElemT>, PCollectionView<ViewT>> delegate() {
    * to {@link ViewT}.
    */
   static final class WriteView<ElemT, ViewT>
-      extends PTransform<PCollection<Iterable<ElemT>>, PCollectionView<ViewT>> {
+      extends RawPTransform<PCollection<Iterable<ElemT>>, PCollectionView<ViewT>, Message> {
     private final CreatePCollectionView<ElemT, ViewT> og;
 
     WriteView(CreatePCollectionView<ElemT, ViewT> og) {
@@ -110,5 +112,13 @@ public PCollectionView<ViewT> expand(PCollection<Iterable<ElemT>> input) {
     public PCollectionView<ViewT> getView() {
       return og.getView();
     }
+
+    @Override
+    public String getUrn() {
+      return DIRECT_WRITE_VIEW_URN;
+    }
   }
+
+  public static final String DIRECT_WRITE_VIEW_URN =
+      "urn:beam:directrunner:transforms:write_view:v1";
 }
diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
index 8ff0cb02eb3c..3430750d37a8 100644
--- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
@@ -405,4 +405,11 @@
     <Bug pattern="NM_CLASS_NOT_EXCEPTION"/>
     <!-- It is clear from the name that this class holds either StorageObject or IOException. -->
   </Match>
+
+  <Match>
+    <Class name="org.apache.beam.runners.direct.ParDoMultiOverrideFactory$StatefulParDo"/>
+    <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED"/>
+    <!-- PTransforms do not actually support serialization. -->
+  </Match>
+
 </FindBugsFilter>

From 6b21ea660eb82c486d17c5d8041c90c6479a54dc Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 25 May 2017 10:30:44 -0700
Subject: [PATCH 339/387] More robust gen_protos on jenkins.

Don't use the globally shared build directory, which pip may refuse
to use if there's a failed install.
---
 sdks/python/gen_protos.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
index edaaff488dfb..bb8592d985c3 100644
--- a/sdks/python/gen_protos.py
+++ b/sdks/python/gen_protos.py
@@ -22,6 +22,7 @@
 import multiprocessing
 import os
 import pkg_resources
+import shutil
 import subprocess
 import sys
 import warnings
@@ -55,10 +56,11 @@ def generate_proto_files():
   if out_files and not proto_files:
     # We have out_files but no protos; assume they're up to date.
     # This is actually the common case (e.g. installation from an sdist).
+    logging.info('No proto files; using existing generated files.')
     return
 
   elif not out_files and not proto_files:
-    if not common:
+    if not os.path.exists(common):
       raise RuntimeError(
           'Not in apache git tree; unable to find proto definitions.')
     else:
@@ -105,9 +107,13 @@ def generate_proto_files():
 def _install_grpcio_tools_and_generate_proto_files():
   install_path = os.path.join(
       os.path.dirname(os.path.abspath(__file__)), '.eggs', 'grpcio-wheels')
-  logging.warning('Downloading a grpcio-tools to %s' % install_path)
+  build_path = install_path + '-build'
+  if os.path.exists(build_path):
+    shutil.rmtree(build_path)
+  logging.warning('Installing grpcio-tools into %s' % install_path)
   subprocess.check_call(
-      ['pip', 'install', '-t', install_path, '--upgrade', GRPC_TOOLS])
+      ['pip', 'install', '--target', install_path, '--build', build_path,
+       '--upgrade', GRPC_TOOLS])
   sys.path.append(install_path)
   generate_proto_files()
 

From 73fbcba88fcc746695a0174d2e69bec4580c1d23 Mon Sep 17 00:00:00 2001
From: Ben Chambers <bjchambers@users.noreply.github.com>
Date: Tue, 23 May 2017 15:44:48 -0700
Subject: [PATCH 340/387] Clarify that PTransform#expand shouldn't be called

---
 .../java/org/apache/beam/sdk/transforms/PTransform.java   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
index d5df944330b6..58051df78215 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
@@ -176,8 +176,12 @@
 public abstract class PTransform<InputT extends PInput, OutputT extends POutput>
     implements Serializable /* See the note above */, HasDisplayData {
   /**
-   * Applies this {@code PTransform} on the given {@code InputT}, and returns its
-   * {@code Output}.
+   * Override this method to specify how this {@code PTransform} should be expanded
+   * on the given {@code InputT}.
+   *
+   * <p>NOTE: This method should not be called directly. Instead apply the
+   * {@code PTransform} should be applied to the {@code InputT} using the {@code apply}
+   * method.
    *
    * <p>Composite transforms, which are defined in terms of other transforms,
    * should return the output of one of the composed transforms.  Non-composite

From 73305d631255d173a0383673700c211055fc818e Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 25 May 2017 13:43:19 -0700
Subject: [PATCH 341/387] Fixup CombineTranslation

---
 .../beam/runners/core/construction/CombineTranslation.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
index e0b6d5c9abf8..855fba740a02 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
@@ -105,7 +105,7 @@ private static SdkFunctionSpec toProto(GlobalCombineFn<?, ?, ?> combineFn) {
   public static Coder<?> getAccumulatorCoder(
       CombinePayload payload, RunnerApi.Components components) throws IOException {
     String id = payload.getAccumulatorCoderId();
-    return Coders.fromProto(components.getCodersOrThrow(id), components);
+    return CoderTranslation.fromProto(components.getCodersOrThrow(id), components);
   }
 
   public static GlobalCombineFn<?, ?, ?> getCombineFn(CombinePayload payload)

From 281eaab3a0ac3810733b87159c9ea9e82a8480fd Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Fri, 19 May 2017 16:09:13 -0700
Subject: [PATCH 342/387] Cleanups in SimpleDoFnRunner and ParDoEvaluator

- Makes it an error to output to an undeclared output tag,
  instead of effectively silently dropping this data.
- Removes code in SimpleDoFnRunner that, IIRC, assigned windows to
  outputs from bundle methods - which is now obsolete because you
  have to specify the window when outputting from FinishBundle,
  and you can't output from StartBundle at all.
---
 .../beam/runners/core/SimpleDoFnRunner.java   | 387 ++++--------------
 .../beam/runners/direct/ParDoEvaluator.java   |  18 +-
 .../spark/translation/MultiDoFnFunction.java  |   8 +-
 .../translation/TransformTranslator.java      |   1 +
 .../StreamingTransformTranslator.java         |   1 +
 .../apache/beam/sdk/transforms/ParDoTest.java |  76 +---
 6 files changed, 102 insertions(+), 389 deletions(-)

diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 97b0b3307740..7d7babd1397c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -20,11 +20,10 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import javax.annotation.Nullable;
@@ -37,20 +36,13 @@
 import org.apache.beam.sdk.state.Timer;
 import org.apache.beam.sdk.state.TimerSpec;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
-import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
-import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
 import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
 import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
 import org.apache.beam.sdk.util.SystemDoFnInternal;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -73,18 +65,19 @@
  */
 public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
 
+  private final PipelineOptions options;
   /** The {@link DoFn} being run. */
   private final DoFn<InputT, OutputT> fn;
 
   /** The {@link DoFnInvoker} being run. */
   private final DoFnInvoker<InputT, OutputT> invoker;
 
-  /** The context used for running the {@link DoFn}. */
-  private final DoFnContext<InputT, OutputT> context;
-
+  private final SideInputReader sideInputReader;
   private final OutputManager outputManager;
 
   private final TupleTag<OutputT> mainOutputTag;
+  /** The set of known output tags. */
+  private final Set<TupleTag<?>> outputTags;
 
   private final boolean observesWindow;
 
@@ -106,12 +99,16 @@ public SimpleDoFnRunner(
       List<TupleTag<?>> additionalOutputTags,
       StepContext stepContext,
       WindowingStrategy<?, ?> windowingStrategy) {
+    this.options = options;
     this.fn = fn;
     this.signature = DoFnSignatures.getSignature(fn.getClass());
     this.observesWindow = signature.processElement().observesWindow() || !sideInputReader.isEmpty();
     this.invoker = DoFnInvokers.invokerFor(fn);
+    this.sideInputReader = sideInputReader;
     this.outputManager = outputManager;
     this.mainOutputTag = mainOutputTag;
+    this.outputTags =
+        Sets.newHashSet(FluentIterable.<TupleTag<?>>of(mainOutputTag).append(additionalOutputTags));
     this.stepContext = stepContext;
 
     // This is a cast of an _invariant_ coder. But we are assured by pipeline validation
@@ -121,26 +118,13 @@ public SimpleDoFnRunner(
         (Coder<BoundedWindow>) windowingStrategy.getWindowFn().windowCoder();
     this.windowCoder = untypedCoder;
     this.allowedLateness = windowingStrategy.getAllowedLateness();
-
-    this.context =
-        new DoFnContext<>(
-            options,
-            fn,
-            sideInputReader,
-            outputManager,
-            mainOutputTag,
-            additionalOutputTags,
-            stepContext,
-            windowingStrategy.getWindowFn());
   }
 
   @Override
   public void startBundle() {
-    DoFnStartBundleContext<InputT, OutputT> startBundleContext =
-        createStartBundleContext(fn, context);
     // This can contain user code. Wrap it in case it throws an exception.
     try {
-      invoker.invokeStartBundle(startBundleContext);
+      invoker.invokeStartBundle(new DoFnStartBundleContext());
     } catch (Throwable t) {
       // Exception in user code.
       throw wrapUserCodeException(t);
@@ -174,7 +158,7 @@ public void onTimer(
 
       case PROCESSING_TIME:
       case SYNCHRONIZED_PROCESSING_TIME:
-        effectiveTimestamp = context.stepContext.timerInternals().currentInputWatermarkTime();
+        effectiveTimestamp = stepContext.timerInternals().currentInputWatermarkTime();
         break;
 
       default:
@@ -182,18 +166,15 @@ public void onTimer(
             String.format("Unknown time domain: %s", timeDomain));
     }
 
-    OnTimerArgumentProvider<InputT, OutputT> argumentProvider =
-        new OnTimerArgumentProvider<>(
-            fn, context, window, allowedLateness, effectiveTimestamp, timeDomain);
+    OnTimerArgumentProvider argumentProvider =
+        new OnTimerArgumentProvider(window, effectiveTimestamp, timeDomain);
     invoker.invokeOnTimer(timerId, argumentProvider);
   }
 
   private void invokeProcessElement(WindowedValue<InputT> elem) {
-    final DoFnProcessContext<InputT, OutputT> processContext = createProcessContext(elem);
-
     // This can contain user code. Wrap it in case it throws an exception.
     try {
-      invoker.invokeProcessElement(processContext);
+      invoker.invokeProcessElement(new DoFnProcessContext(elem));
     } catch (Exception ex) {
       throw wrapUserCodeException(ex);
     }
@@ -201,32 +182,15 @@ private void invokeProcessElement(WindowedValue<InputT> elem) {
 
   @Override
   public void finishBundle() {
-    DoFnFinishBundleContext<InputT, OutputT> finishBundleContext =
-        createFinishBundleContext(fn, context);
     // This can contain user code. Wrap it in case it throws an exception.
     try {
-      invoker.invokeFinishBundle(finishBundleContext);
+      invoker.invokeFinishBundle(new DoFnFinishBundleContext());
     } catch (Throwable t) {
       // Exception in user code.
       throw wrapUserCodeException(t);
     }
   }
 
-  private DoFnStartBundleContext<InputT, OutputT> createStartBundleContext(
-      DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
-    return new DoFnStartBundleContext<>(fn, context);
-  }
-
-  private DoFnFinishBundleContext<InputT, OutputT> createFinishBundleContext(
-      DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
-    return new DoFnFinishBundleContext<>(fn, context);
-  }
-
-  /** Returns a new {@link DoFn.ProcessContext} for the given element. */
-  private DoFnProcessContext<InputT, OutputT> createProcessContext(WindowedValue<InputT> elem) {
-    return new DoFnProcessContext<InputT, OutputT>(fn, context, elem, allowedLateness);
-  }
-
   private RuntimeException wrapUserCodeException(Throwable t) {
     throw UserCodeException.wrapIf(!isSystemDoFn(), t);
   }
@@ -235,170 +199,31 @@ private boolean isSystemDoFn() {
     return invoker.getClass().isAnnotationPresent(SystemDoFnInternal.class);
   }
 
-  /**
-   * A concrete implementation of {@code DoFn.Context} used for running a {@link DoFn}.
-   *
-   * @param <InputT> the type of the {@link DoFn} (main) input elements
-   * @param <OutputT> the type of the {@link DoFn} (main) output elements
-   */
-  private static class DoFnContext<InputT, OutputT> {
-    private static final int MAX_SIDE_OUTPUTS = 1000;
-
-    final PipelineOptions options;
-    final DoFn<InputT, OutputT> fn;
-    final SideInputReader sideInputReader;
-    final OutputManager outputManager;
-    final TupleTag<OutputT> mainOutputTag;
-    final StepContext stepContext;
-    final WindowFn<?, ?> windowFn;
-
-    /**
-     * The set of known output tags, some of which may be undeclared, so we can throw an exception
-     * when it exceeds {@link #MAX_SIDE_OUTPUTS}.
-     */
-    private Set<TupleTag<?>> outputTags;
-
-    public DoFnContext(
-        PipelineOptions options,
-        DoFn<InputT, OutputT> fn,
-        SideInputReader sideInputReader,
-        OutputManager outputManager,
-        TupleTag<OutputT> mainOutputTag,
-        List<TupleTag<?>> additionalOutputTags,
-        StepContext stepContext,
-        WindowFn<?, ?> windowFn) {
-      this.options = options;
-      this.fn = fn;
-      this.sideInputReader = sideInputReader;
-      this.outputManager = outputManager;
-      this.mainOutputTag = mainOutputTag;
-      this.outputTags = Sets.newHashSet();
-
-      outputTags.add(mainOutputTag);
-      for (TupleTag<?> additionalOutputTag : additionalOutputTags) {
-        outputTags.add(additionalOutputTag);
-      }
-
-      this.stepContext = stepContext;
-      this.windowFn = windowFn;
-    }
-
-    //////////////////////////////////////////////////////////////////////////////
-
-    public PipelineOptions getPipelineOptions() {
-      return options;
-    }
-
-    <T, W extends BoundedWindow> WindowedValue<T> makeWindowedValue(
-        T output, Instant timestamp, Collection<W> windows, PaneInfo pane) {
-      final Instant inputTimestamp = timestamp;
-
-      if (timestamp == null) {
-        timestamp = BoundedWindow.TIMESTAMP_MIN_VALUE;
-      }
-
-      if (windows == null) {
-        try {
-          // The windowFn can never succeed at accessing the element, so its type does not
-          // matter here
-          @SuppressWarnings("unchecked")
-          WindowFn<Object, W> objectWindowFn = (WindowFn<Object, W>) windowFn;
-          windows =
-              objectWindowFn.assignWindows(
-                  objectWindowFn.new AssignContext() {
-                    @Override
-                    public Object element() {
-                      throw new UnsupportedOperationException(
-                          "WindowFn attempted to access input element when none was available");
-                    }
-
-                    @Override
-                    public Instant timestamp() {
-                      if (inputTimestamp == null) {
-                        throw new UnsupportedOperationException(
-                            "WindowFn attempted to access input timestamp when none was available");
-                      }
-                      return inputTimestamp;
-                    }
-
-                    @Override
-                    public W window() {
-                      throw new UnsupportedOperationException(
-                          "WindowFn attempted to access input windows when none were available");
-                    }
-                  });
-        } catch (Exception e) {
-          throw UserCodeException.wrap(e);
-        }
-      }
-
-      return WindowedValue.of(output, timestamp, windows, pane);
-    }
-
-    public <T> T sideInput(PCollectionView<T> view, BoundedWindow sideInputWindow) {
-      if (!sideInputReader.contains(view)) {
-        throw new IllegalArgumentException("calling sideInput() with unknown view");
-      }
-      return sideInputReader.get(view, sideInputWindow);
-    }
-
-    void outputWindowedValue(
-        OutputT output,
-        Instant timestamp,
-        Collection<? extends BoundedWindow> windows,
-        PaneInfo pane) {
-      outputWindowedValue(makeWindowedValue(output, timestamp, windows, pane));
-    }
-
-    void outputWindowedValue(WindowedValue<OutputT> windowedElem) {
-      outputManager.output(mainOutputTag, windowedElem);
-    }
-
-    private <T> void outputWindowedValue(
-        TupleTag<T> tag,
-        T output,
-        Instant timestamp,
-        Collection<? extends BoundedWindow> windows,
-        PaneInfo pane) {
-      outputWindowedValue(tag, makeWindowedValue(output, timestamp, windows, pane));
-    }
-
-    private <T> void outputWindowedValue(TupleTag<T> tag, WindowedValue<T> windowedElem) {
-      if (!outputTags.contains(tag)) {
-        // This tag wasn't declared nor was it seen before during this execution.
-        // Thus, this must be a new, undeclared and unconsumed output.
-        // To prevent likely user errors, enforce the limit on the number of side
-        // outputs.
-        if (outputTags.size() >= MAX_SIDE_OUTPUTS) {
-          throw new IllegalArgumentException(
-              "the number of outputs has exceeded a limit of " + MAX_SIDE_OUTPUTS);
-        }
-        outputTags.add(tag);
-      }
-
-      outputManager.output(tag, windowedElem);
+  private <T> T sideInput(PCollectionView<T> view, BoundedWindow sideInputWindow) {
+    if (!sideInputReader.contains(view)) {
+      throw new IllegalArgumentException("calling sideInput() with unknown view");
     }
+    return sideInputReader.get(view, sideInputWindow);
   }
 
+  private <T> void outputWindowedValue(TupleTag<T> tag, WindowedValue<T> windowedElem) {
+    checkArgument(outputTags.contains(tag), "Unknown output tag %s", tag);
+    outputManager.output(tag, windowedElem);
+  }
 
   /**
    * A concrete implementation of {@link DoFn.StartBundleContext}.
    */
-  private class DoFnStartBundleContext<InputT, OutputT>
+  private class DoFnStartBundleContext
       extends DoFn<InputT, OutputT>.StartBundleContext
       implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-    private final DoFn<InputT, OutputT> fn;
-    private final DoFnContext<InputT, OutputT> context;
-
-    private DoFnStartBundleContext(DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+    private DoFnStartBundleContext() {
       fn.super();
-      this.fn = fn;
-      this.context = context;
     }
 
     @Override
     public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
+      return options;
     }
 
     @Override
@@ -408,24 +233,25 @@ public BoundedWindow window() {
     }
 
     @Override
-    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
       return this;
     }
 
     @Override
-    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+        DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
           "Cannot access FinishBundleContext outside of @FinishBundle method.");
     }
 
     @Override
-    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
           "Cannot access ProcessContext outside of @ProcessElement method.");
     }
 
     @Override
-    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
           "Cannot access OnTimerContext outside of @OnTimer methods.");
     }
@@ -453,20 +279,16 @@ public Timer timer(String timerId) {
    * B
    * A concrete implementation of {@link DoFn.FinishBundleContext}.
    */
-  private class DoFnFinishBundleContext<InputT, OutputT>
+  private class DoFnFinishBundleContext
       extends DoFn<InputT, OutputT>.FinishBundleContext
       implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-    private final DoFnContext<InputT, OutputT> context;
-
-    private DoFnFinishBundleContext(
-        DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+    private DoFnFinishBundleContext() {
       fn.super();
-      this.context = context;
     }
 
     @Override
     public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
+      return options;
     }
 
     @Override
@@ -476,24 +298,25 @@ public BoundedWindow window() {
     }
 
     @Override
-    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
           "Cannot access StartBundleContext outside of @StartBundle method.");
     }
 
     @Override
-    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+        DoFn<InputT, OutputT> doFn) {
       return this;
     }
 
     @Override
-    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
           "Cannot access ProcessContext outside of @ProcessElement method.");
     }
 
     @Override
-    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
           "Cannot access OnTimerContext outside of @OnTimer methods.");
     }
@@ -518,30 +341,22 @@ public Timer timer(String timerId) {
 
     @Override
     public void output(OutputT output, Instant timestamp, BoundedWindow window) {
-      context.outputWindowedValue(WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+      output(mainOutputTag, output, timestamp, window);
     }
 
     @Override
     public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
-      context.outputWindowedValue(
-          tag, WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+      outputWindowedValue(tag, WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
     }
   }
 
   /**
    * A concrete implementation of {@link DoFn.ProcessContext} used for running a {@link DoFn} over a
    * single element.
-   *
-   * @param <InputT> the type of the {@link DoFn} (main) input elements
-   * @param <OutputT> the type of the {@link DoFn} (main) output elements
    */
-  private class DoFnProcessContext<InputT, OutputT> extends DoFn<InputT, OutputT>.ProcessContext
+  private class DoFnProcessContext extends DoFn<InputT, OutputT>.ProcessContext
       implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
-    final DoFn<InputT, OutputT> fn;
-    final DoFnContext<InputT, OutputT> context;
-    final WindowedValue<InputT> windowedValue;
-    private final Duration allowedLateness;
+    final WindowedValue<InputT> elem;
 
     /** Lazily initialized; should only be accessed via {@link #getNamespace()}. */
     @Nullable private StateNamespace namespace;
@@ -549,7 +364,7 @@ private class DoFnProcessContext<InputT, OutputT> extends DoFn<InputT, OutputT>.
     /**
      * The state namespace for this context.
      *
-     * <p>Any call to {@link #getNamespace()} when more than one window is present will crash; this
+     * <p>Any call to this method when more than one window is present will crash; this
      * represents a bug in the runner or the {@link DoFnSignature}, since values must be in exactly
      * one window when state or timers are relevant.
      */
@@ -561,55 +376,32 @@ private StateNamespace getNamespace() {
     }
 
     private DoFnProcessContext(
-        DoFn<InputT, OutputT> fn,
-        DoFnContext<InputT, OutputT> context,
-        WindowedValue<InputT> windowedValue,
-        Duration allowedLateness) {
+        WindowedValue<InputT> elem) {
       fn.super();
-      this.fn = fn;
-      this.context = context;
-      this.windowedValue = windowedValue;
-      this.allowedLateness = allowedLateness;
+      this.elem = elem;
     }
 
     @Override
     public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
+      return options;
     }
 
     @Override
     public InputT element() {
-      return windowedValue.getValue();
+      return elem.getValue();
     }
 
     @Override
     public <T> T sideInput(PCollectionView<T> view) {
       checkNotNull(view, "View passed to sideInput cannot be null");
-      Iterator<? extends BoundedWindow> windowIter = windows().iterator();
-      BoundedWindow window;
-      if (!windowIter.hasNext()) {
-        if (context.windowFn instanceof GlobalWindows) {
-          // TODO: Remove this once GroupByKeyOnly no longer outputs elements
-          // without windows
-          window = GlobalWindow.INSTANCE;
-        } else {
-          throw new IllegalStateException(
-              "sideInput called when main input element is not in any windows");
-        }
-      } else {
-        window = windowIter.next();
-        if (windowIter.hasNext()) {
-          throw new IllegalStateException(
-              "sideInput called when main input element is in multiple windows");
-        }
-      }
-      return context.sideInput(
+      BoundedWindow window = Iterables.getOnlyElement(windows());
+      return SimpleDoFnRunner.this.sideInput(
           view, view.getWindowMappingFn().getSideInputWindow(window));
     }
 
     @Override
     public PaneInfo pane() {
-      return windowedValue.getPane();
+      return elem.getPane();
     }
 
     @Override
@@ -619,37 +411,36 @@ public void updateWatermark(Instant watermark) {
 
     @Override
     public void output(OutputT output) {
-      context.outputWindowedValue(windowedValue.withValue(output));
+      output(mainOutputTag, output);
     }
 
     @Override
     public void outputWithTimestamp(OutputT output, Instant timestamp) {
       checkTimestamp(timestamp);
-      context.outputWindowedValue(
-          output, timestamp, windowedValue.getWindows(), windowedValue.getPane());
+      outputWithTimestamp(mainOutputTag, output, timestamp);
     }
 
     @Override
     public <T> void output(TupleTag<T> tag, T output) {
       checkNotNull(tag, "Tag passed to output cannot be null");
-      context.outputWindowedValue(tag, windowedValue.withValue(output));
+      outputWindowedValue(tag, elem.withValue(output));
     }
 
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
       checkNotNull(tag, "Tag passed to outputWithTimestamp cannot be null");
       checkTimestamp(timestamp);
-      context.outputWindowedValue(
-          tag, output, timestamp, windowedValue.getWindows(), windowedValue.getPane());
+      outputWindowedValue(
+          tag, WindowedValue.of(output, timestamp, elem.getWindows(), elem.getPane()));
     }
 
     @Override
     public Instant timestamp() {
-      return windowedValue.getTimestamp();
+      return elem.getTimestamp();
     }
 
     public Collection<? extends BoundedWindow> windows() {
-      return windowedValue.getWindows();
+      return elem.getWindows();
     }
 
     @SuppressWarnings("deprecation") // Allowed Skew is deprecated for users, but must be respected
@@ -657,7 +448,7 @@ private void checkTimestamp(Instant timestamp) {
       // The documentation of getAllowedTimestampSkew explicitly permits Long.MAX_VALUE to be used
       // for infinite skew. Defend against underflow in that case for timestamps before the epoch
       if (fn.getAllowedTimestampSkew().getMillis() != Long.MAX_VALUE
-          && timestamp.isBefore(windowedValue.getTimestamp().minus(fn.getAllowedTimestampSkew()))) {
+          && timestamp.isBefore(elem.getTimestamp().minus(fn.getAllowedTimestampSkew()))) {
         throw new IllegalArgumentException(
             String.format(
                 "Cannot output with timestamp %s. Output timestamps must be no earlier than the "
@@ -665,23 +456,24 @@ private void checkTimestamp(Instant timestamp) {
                     + "DoFn#getAllowedTimestampSkew() Javadoc for details on changing the allowed "
                     + "skew.",
                 timestamp,
-                windowedValue.getTimestamp(),
+                elem.getTimestamp(),
                 PeriodFormat.getDefault().print(fn.getAllowedTimestampSkew().toPeriod())));
       }
     }
 
     @Override
     public BoundedWindow window() {
-      return Iterables.getOnlyElement(windowedValue.getWindows());
+      return Iterables.getOnlyElement(elem.getWindows());
     }
 
     @Override
-    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
     }
 
     @Override
-    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+        DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
     }
 
@@ -691,7 +483,7 @@ public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT>
     }
 
     @Override
-    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException(
           "Cannot access OnTimerContext outside of @OnTimer methods.");
     }
@@ -720,7 +512,7 @@ public Timer timer(String timerId) {
         TimerSpec spec =
             (TimerSpec) signature.timerDeclarations().get(timerId).field().get(fn);
         return new TimerInternalsTimer(
-            window(), getNamespace(), allowedLateness, timerId, spec, stepContext.timerInternals());
+            window(), getNamespace(), timerId, spec, stepContext.timerInternals());
       } catch (IllegalAccessException e) {
         throw new RuntimeException(e);
       }
@@ -730,20 +522,13 @@ public Timer timer(String timerId) {
   /**
    * A concrete implementation of {@link DoFnInvoker.ArgumentProvider} used for running a {@link
    * DoFn} on a timer.
-   *
-   * @param <InputT> the type of the {@link DoFn} (main) input elements
-   * @param <OutputT> the type of the {@link DoFn} (main) output elements
    */
-  private class OnTimerArgumentProvider<InputT, OutputT>
+  private class OnTimerArgumentProvider
       extends DoFn<InputT, OutputT>.OnTimerContext
       implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
-    final DoFn<InputT, OutputT> fn;
-    final DoFnContext<InputT, OutputT> context;
     private final BoundedWindow window;
     private final Instant timestamp;
     private final TimeDomain timeDomain;
-    private final Duration allowedLateness;
 
     /** Lazily initialized; should only be accessed via {@link #getNamespace()}. */
     private StateNamespace namespace;
@@ -751,7 +536,7 @@ private class OnTimerArgumentProvider<InputT, OutputT>
     /**
      * The state namespace for this context.
      *
-     * <p>Any call to {@link #getNamespace()} when more than one window is present will crash; this
+     * <p>Any call to this method when more than one window is present will crash; this
      * represents a bug in the runner or the {@link DoFnSignature}, since values must be in exactly
      * one window when state or timers are relevant.
      */
@@ -763,17 +548,11 @@ private StateNamespace getNamespace() {
     }
 
     private OnTimerArgumentProvider(
-        DoFn<InputT, OutputT> fn,
-        DoFnContext<InputT, OutputT> context,
         BoundedWindow window,
-        Duration allowedLateness,
         Instant timestamp,
         TimeDomain timeDomain) {
       fn.super();
-      this.fn = fn;
-      this.context = context;
       this.window = window;
-      this.allowedLateness = allowedLateness;
       this.timestamp = timestamp;
       this.timeDomain = timeDomain;
     }
@@ -789,12 +568,13 @@ public BoundedWindow window() {
     }
 
     @Override
-    public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
     }
 
     @Override
-    public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+        DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
     }
 
@@ -805,12 +585,12 @@ public TimeDomain timeDomain() {
 
 
     @Override
-    public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
       throw new UnsupportedOperationException("ProcessContext parameters are not supported.");
     }
 
     @Override
-    public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+    public DoFn<InputT, OutputT>.OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
       return this;
     }
 
@@ -838,7 +618,7 @@ public Timer timer(String timerId) {
         TimerSpec spec =
             (TimerSpec) signature.timerDeclarations().get(timerId).field().get(fn);
         return new TimerInternalsTimer(
-            window, getNamespace(), allowedLateness, timerId, spec, stepContext.timerInternals());
+            window, getNamespace(), timerId, spec, stepContext.timerInternals());
       } catch (IllegalAccessException e) {
         throw new RuntimeException(e);
       }
@@ -846,42 +626,37 @@ public Timer timer(String timerId) {
 
     @Override
     public PipelineOptions getPipelineOptions() {
-      return context.getPipelineOptions();
+      return options;
     }
 
     @Override
     public void output(OutputT output) {
-      context.outputWindowedValue(
-          output, timestamp(), Collections.singleton(window()), PaneInfo.NO_FIRING);
+      output(mainOutputTag, output);
     }
 
     @Override
     public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      context.outputWindowedValue(
-          output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+      outputWithTimestamp(mainOutputTag, output, timestamp);
     }
 
     @Override
     public <T> void output(TupleTag<T> tag, T output) {
-      context.outputWindowedValue(
-          tag, output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+      outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
     }
 
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      context.outputWindowedValue(
-          tag, output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+      outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
     }
   }
 
-  private static class TimerInternalsTimer implements Timer {
+  private class TimerInternalsTimer implements Timer {
     private final TimerInternals timerInternals;
 
     // The window and the namespace represent the same thing, but the namespace is a cached
     // and specially encoded form. Since the namespace can be cached across timers, it is
     // passed in whole rather than being computed here.
     private final BoundedWindow window;
-    private final Duration allowedLateness;
     private final StateNamespace namespace;
     private final String timerId;
     private final TimerSpec spec;
@@ -891,12 +666,10 @@ private static class TimerInternalsTimer implements Timer {
     public TimerInternalsTimer(
         BoundedWindow window,
         StateNamespace namespace,
-        Duration allowedLateness,
         String timerId,
         TimerSpec spec,
         TimerInternals timerInternals) {
       this.window = window;
-      this.allowedLateness = allowedLateness;
       this.namespace = namespace;
       this.timerId = timerId;
       this.spec = spec;
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
index 28fc68d293cb..26da6c656ade 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
@@ -17,8 +17,9 @@
  */
 package org.apache.beam.runners.direct;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import com.google.common.collect.ImmutableList;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -218,7 +219,6 @@ public TransformResult<InputT> finishBundle() {
 
   static class BundleOutputManager implements OutputManager {
     private final Map<TupleTag<?>, UncommittedBundle<?>> bundles;
-    private final Map<TupleTag<?>, List<?>> undeclaredOutputs;
 
     public static BundleOutputManager create(Map<TupleTag<?>, UncommittedBundle<?>> outputBundles) {
       return new BundleOutputManager(outputBundles);
@@ -226,23 +226,13 @@ public static BundleOutputManager create(Map<TupleTag<?>, UncommittedBundle<?>>
 
     private BundleOutputManager(Map<TupleTag<?>, UncommittedBundle<?>> bundles) {
       this.bundles = bundles;
-      undeclaredOutputs = new HashMap<>();
     }
 
     @SuppressWarnings({"unchecked", "rawtypes"})
     @Override
     public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
-      UncommittedBundle bundle = bundles.get(tag);
-      if (bundle == null) {
-        List<WindowedValue<T>> undeclaredContents = (List) undeclaredOutputs.get(tag);
-        if (undeclaredContents == null) {
-          undeclaredContents = new ArrayList<>();
-          undeclaredOutputs.put(tag, undeclaredContents);
-        }
-        undeclaredContents.add(output);
-      } else {
-        bundle.add(output);
-      }
+      checkArgument(bundles.containsKey(tag), "Unknown output tag %s", tag);
+      ((UncommittedBundle) bundles.get(tag)).add(output);
     }
   }
 }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
index 4a665417a078..32749122d5b2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java
@@ -22,8 +22,8 @@
 import com.google.common.collect.Iterators;
 import com.google.common.collect.LinkedListMultimap;
 import com.google.common.collect.Multimap;
-import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import org.apache.beam.runners.core.DoFnRunner;
 import org.apache.beam.runners.core.DoFnRunners;
@@ -57,6 +57,7 @@ public class MultiDoFnFunction<InputT, OutputT>
   private final DoFn<InputT, OutputT> doFn;
   private final SparkRuntimeContext runtimeContext;
   private final TupleTag<OutputT> mainOutputTag;
+  private final List<TupleTag<?>> additionalOutputTags;
   private final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs;
   private final WindowingStrategy<?, ?> windowingStrategy;
 
@@ -66,6 +67,7 @@ public class MultiDoFnFunction<InputT, OutputT>
    * @param doFn              The {@link DoFn} to be wrapped.
    * @param runtimeContext    The {@link SparkRuntimeContext}.
    * @param mainOutputTag     The main output {@link TupleTag}.
+   * @param additionalOutputTags Additional {@link TupleTag output tags}.
    * @param sideInputs        Side inputs used in this {@link DoFn}.
    * @param windowingStrategy Input {@link WindowingStrategy}.
    */
@@ -76,6 +78,7 @@ public MultiDoFnFunction(
       DoFn<InputT, OutputT> doFn,
       SparkRuntimeContext runtimeContext,
       TupleTag<OutputT> mainOutputTag,
+      List<TupleTag<?>> additionalOutputTags,
       Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs,
       WindowingStrategy<?, ?> windowingStrategy) {
     this.aggAccum = aggAccum;
@@ -84,6 +87,7 @@ public MultiDoFnFunction(
     this.doFn = doFn;
     this.runtimeContext = runtimeContext;
     this.mainOutputTag = mainOutputTag;
+    this.additionalOutputTags = additionalOutputTags;
     this.sideInputs = sideInputs;
     this.windowingStrategy = windowingStrategy;
   }
@@ -101,7 +105,7 @@ public Iterable<Tuple2<TupleTag<?>, WindowedValue<?>>> call(
             new SparkSideInputReader(sideInputs),
             outputManager,
             mainOutputTag,
-            Collections.<TupleTag<?>>emptyList(),
+            additionalOutputTags,
             new SparkProcessContext.NoOpStepContext(),
             windowingStrategy);
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index b2ed3a9265d6..742ea83539e2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -369,6 +369,7 @@ public void evaluate(
                     doFn,
                     context.getRuntimeContext(),
                     transform.getMainOutputTag(),
+                    transform.getAdditionalOutputTags().getAll(),
                     TranslationUtils.getSideInputs(transform.getSideInputs(), context),
                     windowingStrategy));
         Map<TupleTag<?>, PValue> outputs = context.getOutputs(transform);
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index acb4a02f1e94..43f4b756d186 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -411,6 +411,7 @@ public JavaPairRDD<TupleTag<?>, WindowedValue<?>> call(
                             doFn,
                             runtimeContext,
                             transform.getMainOutputTag(),
+                            transform.getAdditionalOutputTags().getAll(),
                             sideInputs,
                             windowingStrategy));
                   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 35c02ba8c9d0..c67cf2a758fe 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -32,7 +32,6 @@
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
-import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
@@ -515,58 +514,18 @@ public void processElement(ProcessContext c) {
   @Test
   @Category(NeedsRunner.class)
   public void testParDoWritingToUndeclaredTag() {
-
     List<Integer> inputs = Arrays.asList(3, -42, 666);
 
     TupleTag<String> notOutputTag = new TupleTag<String>("additional"){};
 
-    PCollection<String> output = pipeline
+    pipeline
         .apply(Create.of(inputs))
         .apply(ParDo.of(new TestDoFn(
             Arrays.<PCollectionView<Integer>>asList(),
-            Arrays.asList(notOutputTag))));
-
-    PAssert.that(output)
-        .satisfies(ParDoTest.HasExpectedOutput.forInput(inputs));
+            Arrays.asList(notOutputTag)))
+            /* No call to .withOutputTags - should cause error */);
 
-    pipeline.run();
-  }
-
-  @Test
-  // TODO: The exception thrown is runner-specific, even if the behavior is general
-  @Category(NeedsRunner.class)
-  public void testParDoUndeclaredTagLimit() {
-
-    PCollection<Integer> input = pipeline.apply(Create.of(Arrays.asList(3)));
-
-    // Success for a total of 1000 outputs.
-    input
-        .apply("Success1000", ParDo.of(new DoFn<Integer, String>() {
-            @ProcessElement
-            public void processElement(ProcessContext c) {
-              TupleTag<String> specialOutputTag = new TupleTag<String>(){};
-              c.output(specialOutputTag, "special");
-              c.output(specialOutputTag, "special");
-              c.output(specialOutputTag, "special");
-
-              for (int i = 0; i < 998; i++) {
-                c.output(new TupleTag<String>(){}, "tag" + i);
-              }
-            }}));
-    pipeline.run();
-
-    // Failure for a total of 1001 outputs.
-    input
-        .apply("Failure1001", ParDo.of(new DoFn<Integer, String>() {
-            @ProcessElement
-            public void processElement(ProcessContext c) {
-              for (int i = 0; i < 1000; i++) {
-                c.output(new TupleTag<String>(){}, "output" + i);
-              }
-            }}));
-
-    thrown.expect(RuntimeException.class);
-    thrown.expectMessage("the number of outputs has exceeded a limit");
+    thrown.expectMessage("additional");
     pipeline.run();
   }
 
@@ -1107,43 +1066,32 @@ static class HasExpectedOutput
     private final List<Integer> inputs;
     private final List<Integer> sideInputs;
     private final String additionalOutput;
-    private final boolean ordered;
 
     public static HasExpectedOutput forInput(List<Integer> inputs) {
       return new HasExpectedOutput(
           new ArrayList<Integer>(inputs),
           new ArrayList<Integer>(),
-          "",
-          false);
+          "");
     }
 
     private HasExpectedOutput(List<Integer> inputs,
                               List<Integer> sideInputs,
-                              String additionalOutput,
-                              boolean ordered) {
+                              String additionalOutput) {
       this.inputs = inputs;
       this.sideInputs = sideInputs;
       this.additionalOutput = additionalOutput;
-      this.ordered = ordered;
     }
 
     public HasExpectedOutput andSideInputs(Integer... sideInputValues) {
-      List<Integer> sideInputs = new ArrayList<>();
-      for (Integer sideInputValue : sideInputValues) {
-        sideInputs.add(sideInputValue);
-      }
-      return new HasExpectedOutput(inputs, sideInputs, additionalOutput, ordered);
+      return new HasExpectedOutput(
+          inputs, Arrays.asList(sideInputValues), additionalOutput);
     }
 
     public HasExpectedOutput fromOutput(TupleTag<String> outputTag) {
       return fromOutput(outputTag.getId());
     }
     public HasExpectedOutput fromOutput(String outputId) {
-      return new HasExpectedOutput(inputs, sideInputs, outputId, ordered);
-    }
-
-    public HasExpectedOutput inOrder() {
-      return new HasExpectedOutput(inputs, sideInputs, additionalOutput, true);
+      return new HasExpectedOutput(inputs, sideInputs, outputId);
     }
 
     @Override
@@ -1179,11 +1127,7 @@ public Void apply(Iterable<String> outputs) {
       }
       String[] expectedProcessedsArray =
           expectedProcesseds.toArray(new String[expectedProcesseds.size()]);
-      if (!ordered || expectedProcesseds.isEmpty()) {
-        assertThat(processeds, containsInAnyOrder(expectedProcessedsArray));
-      } else {
-        assertThat(processeds, contains(expectedProcessedsArray));
-      }
+      assertThat(processeds, containsInAnyOrder(expectedProcessedsArray));
 
       for (String finished : finisheds) {
         assertEquals(additionalOutputPrefix + "finished", finished);

From b4ad7cb01848262fbe82987beec45efd87649894 Mon Sep 17 00:00:00 2001
From: Eugene Kirpichov <kirpichov@google.com>
Date: Thu, 25 May 2017 13:59:09 -0700
Subject: [PATCH 343/387] Fixes pb2.py path in gitignore

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 6e077bd1d440..bd419a78beec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,7 +25,7 @@ sdks/python/**/*.egg
 sdks/python/LICENSE
 sdks/python/NOTICE
 sdks/python/README.md
-sdks/python/runners/api/*pb2*.*
+sdks/python/apache_beam/runners/api/*pb2*.*
 
 # Ignore IntelliJ files.
 .idea/

From a75f41cc575dba2e5d1c650c0226a91d7ebe125a Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 25 May 2017 17:07:08 -0700
Subject: [PATCH 344/387] Additional explicit file cleanup in gen_protos.

---
 sdks/python/gen_protos.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
index bb8592d985c3..92b8414583ea 100644
--- a/sdks/python/gen_protos.py
+++ b/sdks/python/gen_protos.py
@@ -111,9 +111,12 @@ def _install_grpcio_tools_and_generate_proto_files():
   if os.path.exists(build_path):
     shutil.rmtree(build_path)
   logging.warning('Installing grpcio-tools into %s' % install_path)
-  subprocess.check_call(
-      ['pip', 'install', '--target', install_path, '--build', build_path,
-       '--upgrade', GRPC_TOOLS])
+  try:
+    subprocess.check_call(
+        ['pip', 'install', '--target', install_path, '--build', build_path,
+         '--upgrade', GRPC_TOOLS])
+  finally:
+    shutil.rmtree(build_path)
   sys.path.append(install_path)
   generate_proto_files()
 

From 14329b86e0732acf0190e10d9b04c574baaed2e7 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Wed, 24 May 2017 18:14:59 -0700
Subject: [PATCH 345/387] [BEAM-2354] Add a
 ReadStringsFromPubSub/WriteStringsToPubSub PTransform

---
 .../apache_beam/examples/streaming_wordcap.py |  6 +-
 .../examples/streaming_wordcount.py           |  6 +-
 sdks/python/apache_beam/io/gcp/pubsub.py      | 87 ++++++++++++++++---
 sdks/python/apache_beam/io/gcp/pubsub_test.py | 57 ++++++++++--
 .../runners/dataflow/dataflow_runner.py       |  8 +-
 5 files changed, 132 insertions(+), 32 deletions(-)

diff --git a/sdks/python/apache_beam/examples/streaming_wordcap.py b/sdks/python/apache_beam/examples/streaming_wordcap.py
index ce43e1f09849..19f9e5f77ed8 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcap.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcap.py
@@ -44,8 +44,7 @@ def run(argv=None):
   with beam.Pipeline(argv=pipeline_args) as p:
 
     # Read the text file[pattern] into a PCollection.
-    lines = p | beam.io.Read(
-        beam.io.PubSubSource(known_args.input_topic))
+    lines = p | beam.io.ReadStringsFromPubSub(known_args.input_topic)
 
     # Capitalize the characters in each line.
     transformed = (lines
@@ -53,8 +52,7 @@ def run(argv=None):
 
     # Write to PubSub.
     # pylint: disable=expression-not-assigned
-    transformed | beam.io.Write(
-        beam.io.PubSubSink(known_args.output_topic))
+    transformed | beam.io.WriteStringsToPubSub(known_args.output_topic)
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py
index e9d5dbefa835..ed8b5d08dc62 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcount.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcount.py
@@ -47,8 +47,7 @@ def run(argv=None):
   with beam.Pipeline(argv=pipeline_args) as p:
 
     # Read the text file[pattern] into a PCollection.
-    lines = p | 'read' >> beam.io.Read(
-        beam.io.PubSubSource(known_args.input_topic))
+    lines = p | beam.io.ReadStringsFromPubSub(known_args.input_topic)
 
     # Capitalize the characters in each line.
     transformed = (lines
@@ -63,8 +62,7 @@ def run(argv=None):
 
     # Write to PubSub.
     # pylint: disable=expression-not-assigned
-    transformed | 'pubsub_write' >> beam.io.Write(
-        beam.io.PubSubSink(known_args.output_topic))
+    transformed | beam.io.WriteStringsToPubSub(known_args.output_topic)
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index 103fce0b8714..7b838d23fea6 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -22,15 +22,68 @@
 
 from __future__ import absolute_import
 
-from apache_beam import coders
+from apache_beam.io.iobase import Read
+from apache_beam.io.iobase import Write
 from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
+from apache_beam.transforms import PTransform
+from apache_beam.transforms import ParDo
 from apache_beam.transforms.display import DisplayDataItem
 
-__all__ = ['PubSubSink', 'PubSubSource']
 
+__all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub']
 
-class PubSubSource(dataflow_io.NativeSource):
-  """Source for reading from a given Cloud Pub/Sub topic.
+
+class ReadStringsFromPubSub(PTransform):
+  """A ``PTransform`` for reading utf-8 string payloads from Cloud Pub/Sub."""
+
+  def __init__(self, topic, subscription=None, id_label=None):
+    """Initializes ``ReadStringsFromPubSub``.
+
+    Attributes:
+      topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
+      subscription: Optional existing Cloud Pub/Sub subscription to use in the
+        form "projects/<project>/subscriptions/<subscription>".
+      id_label: The attribute on incoming Pub/Sub messages to use as a unique
+        record identifier.  When specified, the value of this attribute (which
+        can be any string that uniquely identifies the record) will be used for
+        deduplication of messages.  If not provided, we cannot guarantee
+        that no duplicate data will be delivered on the Pub/Sub stream. In this
+        case, deduplication of the stream will be strictly best effort.
+    """
+    super(ReadStringsFromPubSub, self).__init__()
+    self._source = _PubSubPayloadSource(
+        topic,
+        subscription=subscription,
+        id_label=id_label)
+
+  def expand(self, pvalue):
+    pcoll = pvalue.pipeline | Read(self._source)
+    pcoll.element_type = bytes
+    pcoll = pcoll | 'decode string' >> ParDo(_decodeUtf8String)
+    pcoll.element_type = unicode
+    return pcoll
+
+
+class WriteStringsToPubSub(PTransform):
+  """A ``PTransform`` for writing utf-8 string payloads to Cloud Pub/Sub."""
+
+  def __init__(self, topic):
+    """Initializes ``WriteStringsToPubSub``.
+
+    Attributes:
+      topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
+    """
+    super(WriteStringsToPubSub, self).__init__()
+    self._sink = _PubSubPayloadSink(topic)
+
+  def expand(self, pcoll):
+    pcoll = pcoll | 'encode string' >> ParDo(_encodeUtf8String)
+    pcoll.element_type = bytes
+    return pcoll | Write(self._sink)
+
+
+class _PubSubPayloadSource(dataflow_io.NativeSource):
+  """Source for the payload of a message as bytes from a Cloud Pub/Sub topic.
 
   Attributes:
     topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
@@ -42,15 +95,12 @@ class PubSubSource(dataflow_io.NativeSource):
       deduplication of messages.  If not provided, Dataflow cannot guarantee
       that no duplicate data will be delivered on the Pub/Sub stream. In this
       case, deduplication of the stream will be strictly best effort.
-    coder: The Coder to use for decoding incoming Pub/Sub messages.
   """
 
-  def __init__(self, topic, subscription=None, id_label=None,
-               coder=coders.StrUtf8Coder()):
+  def __init__(self, topic, subscription=None, id_label=None):
     self.topic = topic
     self.subscription = subscription
     self.id_label = id_label
-    self.coder = coder
 
   @property
   def format(self):
@@ -70,15 +120,14 @@ def display_data(self):
 
   def reader(self):
     raise NotImplementedError(
-        'PubSubSource is not supported in local execution.')
+        'PubSubPayloadSource is not supported in local execution.')
 
 
-class PubSubSink(dataflow_io.NativeSink):
-  """Sink for writing to a given Cloud Pub/Sub topic."""
+class _PubSubPayloadSink(dataflow_io.NativeSink):
+  """Sink for the payload of a message as bytes to a Cloud Pub/Sub topic."""
 
-  def __init__(self, topic, coder=coders.StrUtf8Coder()):
+  def __init__(self, topic):
     self.topic = topic
-    self.coder = coder
 
   @property
   def format(self):
@@ -90,4 +139,14 @@ def display_data(self):
 
   def writer(self):
     raise NotImplementedError(
-        'PubSubSink is not supported in local execution.')
+        'PubSubPayloadSink is not supported in local execution.')
+
+
+def _decodeUtf8String(encoded_value):
+  """Decodes a string in utf-8 format from bytes"""
+  return encoded_value.decode('utf-8')
+
+
+def _encodeUtf8String(value):
+  """Encodes a string in utf-8 format to bytes"""
+  return value.encode('utf-8')
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py
index 1642a95a59e7..322d08a34cb3 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_test.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py
@@ -22,16 +22,53 @@
 
 import hamcrest as hc
 
-from apache_beam.io.gcp.pubsub import PubSubSink
-from apache_beam.io.gcp.pubsub import PubSubSource
+from apache_beam.io.gcp.pubsub import _decodeUtf8String
+from apache_beam.io.gcp.pubsub import _encodeUtf8String
+from apache_beam.io.gcp.pubsub import _PubSubPayloadSink
+from apache_beam.io.gcp.pubsub import _PubSubPayloadSource
+from apache_beam.io.gcp.pubsub import ReadStringsFromPubSub
+from apache_beam.io.gcp.pubsub import WriteStringsToPubSub
+from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.transforms.display import DisplayData
 from apache_beam.transforms.display_test import DisplayDataItemMatcher
 
 
-class TestPubSubSource(unittest.TestCase):
+class TestReadStringsFromPubSub(unittest.TestCase):
+  def test_expand(self):
+    p = TestPipeline()
+    pcoll = p | ReadStringsFromPubSub('a_topic', 'a_subscription', 'a_label')
+    # Ensure that the output type is str
+    self.assertEqual(unicode, pcoll.element_type)
+
+    # Ensure that the type on the intermediate read output PCollection is bytes
+    read_pcoll = pcoll.producer.inputs[0]
+    self.assertEqual(bytes, read_pcoll.element_type)
+
+    # Ensure that the properties passed through correctly
+    source = read_pcoll.producer.transform.source
+    self.assertEqual('a_topic', source.topic)
+    self.assertEqual('a_subscription', source.subscription)
+    self.assertEqual('a_label', source.id_label)
+
+
+class TestWriteStringsToPubSub(unittest.TestCase):
+  def test_expand(self):
+    p = TestPipeline()
+    pdone = p | ReadStringsFromPubSub('baz') | WriteStringsToPubSub('a_topic')
+
+    # Ensure that the properties passed through correctly
+    sink = pdone.producer.transform.sink
+    self.assertEqual('a_topic', sink.topic)
 
+    # Ensure that the type on the intermediate payload transformer output
+    # PCollection is bytes
+    write_pcoll = pdone.producer.inputs[0]
+    self.assertEqual(bytes, write_pcoll.element_type)
+
+
+class TestPubSubSource(unittest.TestCase):
   def test_display_data(self):
-    source = PubSubSource('a_topic', 'a_subscription', 'a_label')
+    source = _PubSubPayloadSource('a_topic', 'a_subscription', 'a_label')
     dd = DisplayData.create_from(source)
     expected_items = [
         DisplayDataItemMatcher('topic', 'a_topic'),
@@ -41,7 +78,7 @@ def test_display_data(self):
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
   def test_display_data_no_subscription(self):
-    source = PubSubSource('a_topic')
+    source = _PubSubPayloadSource('a_topic')
     dd = DisplayData.create_from(source)
     expected_items = [
         DisplayDataItemMatcher('topic', 'a_topic')]
@@ -51,7 +88,7 @@ def test_display_data_no_subscription(self):
 
 class TestPubSubSink(unittest.TestCase):
   def test_display_data(self):
-    sink = PubSubSink('a_topic')
+    sink = _PubSubPayloadSink('a_topic')
     dd = DisplayData.create_from(sink)
     expected_items = [
         DisplayDataItemMatcher('topic', 'a_topic')]
@@ -59,6 +96,14 @@ def test_display_data(self):
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
 
+class TestEncodeDecodeUtf8String(unittest.TestCase):
+  def test_encode(self):
+    self.assertEqual(b'test_data', _encodeUtf8String('test_data'))
+
+  def test_decode(self):
+    self.assertEqual('test_data', _decodeUtf8String(b'test_data'))
+
+
 if __name__ == '__main__':
   logging.getLogger().setLevel(logging.INFO)
   unittest.main()
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index a05e582b05f4..046d3d585388 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -595,8 +595,8 @@ def run_Read(self, transform_node):
       standard_options = (
           transform_node.inputs[0].pipeline.options.view_as(StandardOptions))
       if not standard_options.streaming:
-        raise ValueError('PubSubSource is currently available for use only in '
-                         'streaming pipelines.')
+        raise ValueError('PubSubPayloadSource is currently available for use '
+                         'only in streaming pipelines.')
       step.add_property(PropertyNames.PUBSUB_TOPIC, transform.source.topic)
       if transform.source.subscription:
         step.add_property(PropertyNames.PUBSUB_SUBSCRIPTION,
@@ -677,8 +677,8 @@ def run__NativeWrite(self, transform_node):
       standard_options = (
           transform_node.inputs[0].pipeline.options.view_as(StandardOptions))
       if not standard_options.streaming:
-        raise ValueError('PubSubSink is currently available for use only in '
-                         'streaming pipelines.')
+        raise ValueError('PubSubPayloadSink is currently available for use '
+                         'only in streaming pipelines.')
       step.add_property(PropertyNames.PUBSUB_TOPIC, transform.sink.topic)
     else:
       raise ValueError(

From 4e8c388a46636a8e5391517876ec1adb818b8d61 Mon Sep 17 00:00:00 2001
From: Borisa Zivkovic <borisa.zivkovic@huawei.com>
Date: Mon, 15 May 2017 08:56:19 +0100
Subject: [PATCH 346/387] [BEAM-2276] Add windowing into default filename
 policy

---
 .../beam/sdk/io/DefaultFilenamePolicy.java    | 151 ++++++++++++++++--
 .../java/org/apache/beam/sdk/io/TextIO.java   |   8 +-
 .../sdk/io/DefaultFilenamePolicyTest.java     |  57 +++++++
 .../org/apache/beam/sdk/io/TextIOTest.java    |  10 --
 4 files changed, 196 insertions(+), 30 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java
index 07bc2db4d804..50738546b14e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java
@@ -32,22 +32,58 @@
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
- * A default {@link FilenamePolicy} for unwindowed files. This policy is constructed using three
- * parameters that together define the output name of a sharded file, in conjunction with the number
- * of shards and index of the particular file, using {@link #constructName}.
+ * A default {@link FilenamePolicy} for windowed and unwindowed files. This policy is constructed
+ * using three parameters that together define the output name of a sharded file, in conjunction
+ * with the number of shards, index of the particular file, current window and pane information,
+ * using {@link #constructName}.
  *
- * <p>Most users of unwindowed files will use this {@link DefaultFilenamePolicy}. For more advanced
+ * <p>Most users will use this {@link DefaultFilenamePolicy}. For more advanced
  * uses in generating different files for each window and other sharding controls, see the
  * {@code WriteOneFilePerWindow} example pipeline.
  */
 public final class DefaultFilenamePolicy extends FilenamePolicy {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DefaultFilenamePolicy.class);
+
   /** The default sharding name template used in {@link #constructUsingStandardParameters}. */
   public static final String DEFAULT_SHARD_TEMPLATE = ShardNameTemplate.INDEX_OF_MAX;
 
+  /** The default windowed sharding name template used when writing windowed files.
+   *  This is used as default in cases when user did not specify shard template to
+   *  be used and there is a need to write windowed files. In cases when user does
+   *  specify shard template to be used then provided template will be used for both
+   *  windowed and non-windowed file names.
+   */
+  private static final String DEFAULT_WINDOWED_SHARD_TEMPLATE =
+      "P-W" + DEFAULT_SHARD_TEMPLATE;
+
+  /*
+   * pattern for only non-windowed file names
+   */
+  private static final String NON_WINDOWED_ONLY_PATTERN = "S+|N+";
+
+  /*
+   * pattern for only windowed file names
+   */
+  private static final String WINDOWED_ONLY_PATTERN = "P|W";
+
+  /*
+   * pattern for both windowed and non-windowed file names
+   */
+  private static final String TEMPLATE_PATTERN = "(" + NON_WINDOWED_ONLY_PATTERN + "|"
+   + WINDOWED_ONLY_PATTERN + ")";
+
   // Pattern that matches shard placeholders within a shard template.
-  private static final Pattern SHARD_FORMAT_RE = Pattern.compile("(S+|N+)");
+  private static final Pattern SHARD_FORMAT_RE = Pattern.compile(TEMPLATE_PATTERN);
+  private static final Pattern WINDOWED_FORMAT_RE = Pattern.compile(WINDOWED_ONLY_PATTERN);
 
   /**
    * Constructs a new {@link DefaultFilenamePolicy}.
@@ -69,7 +105,8 @@ public final class DefaultFilenamePolicy extends FilenamePolicy {
    * <p>Any filename component of the provided resource will be used as the filename prefix.
    *
    * <p>If provided, the shard name template will be used; otherwise {@link #DEFAULT_SHARD_TEMPLATE}
-   * will be used.
+   * will be used for non-windowed file names and {@link #DEFAULT_WINDOWED_SHARD_TEMPLATE} will
+   * be used for windowed file names.
    *
    * <p>If provided, the suffix will be used; otherwise the files will have an empty suffix.
    */
@@ -87,6 +124,19 @@ public static DefaultFilenamePolicy constructUsingStandardParameters(
   private final String shardTemplate;
   private final String suffix;
 
+  /*
+   * Checks whether given template contains enough information to form
+   * meaningful windowed file names - ie whether it uses pane and window
+   * info.
+   */
+  static boolean isWindowedTemplate(String template){
+    if (template != null){
+      Matcher m = WINDOWED_FORMAT_RE.matcher(template);
+      return m.find();
+    }
+    return false;
+  }
+
   /**
    * Constructs a fully qualified name from components.
    *
@@ -95,29 +145,45 @@ public static DefaultFilenamePolicy constructUsingStandardParameters(
    * strings.
    *
    * <p>Within a shard template, repeating sequences of the letters "S" or "N"
-   * are replaced with the shard number, or number of shards respectively.  The
-   * numbers are formatted with leading zeros to match the length of the
+   * are replaced with the shard number, or number of shards respectively.
+   * "P" is replaced with by stringification of current pane.
+   * "W" is replaced by stringification of current window.
+   *
+   * <p>The numbers are formatted with leading zeros to match the length of the
    * repeated sequence of letters.
    *
    * <p>For example, if prefix = "output", shardTemplate = "-SSS-of-NNN", and
    * suffix = ".txt", with shardNum = 1 and numShards = 100, the following is
    * produced:  "output-001-of-100.txt".
    */
-  public static String constructName(
-      String prefix, String shardTemplate, String suffix, int shardNum, int numShards) {
+  static String constructName(
+      String prefix, String shardTemplate, String suffix, int shardNum, int numShards,
+      String paneStr, String windowStr) {
     // Matcher API works with StringBuffer, rather than StringBuilder.
     StringBuffer sb = new StringBuffer();
     sb.append(prefix);
 
     Matcher m = SHARD_FORMAT_RE.matcher(shardTemplate);
     while (m.find()) {
-      boolean isShardNum = (m.group(1).charAt(0) == 'S');
+      boolean isCurrentShardNum = (m.group(1).charAt(0) == 'S');
+      boolean isNumberOfShards = (m.group(1).charAt(0) == 'N');
+      boolean isPane = (m.group(1).charAt(0) == 'P') && paneStr != null;
+      boolean isWindow = (m.group(1).charAt(0) == 'W') && windowStr != null;
 
       char[] zeros = new char[m.end() - m.start()];
       Arrays.fill(zeros, '0');
       DecimalFormat df = new DecimalFormat(String.valueOf(zeros));
-      String formatted = df.format(isShardNum ? shardNum : numShards);
-      m.appendReplacement(sb, formatted);
+      if (isCurrentShardNum) {
+        String formatted = df.format(shardNum);
+        m.appendReplacement(sb, formatted);
+      } else if (isNumberOfShards) {
+        String formatted = df.format(numShards);
+        m.appendReplacement(sb, formatted);
+      } else if (isPane) {
+        m.appendReplacement(sb, paneStr);
+      } else if (isWindow) {
+        m.appendReplacement(sb, windowStr);
+      }
     }
     m.appendTail(sb);
 
@@ -125,6 +191,11 @@ public static String constructName(
     return sb.toString();
   }
 
+  static String constructName(String prefix, String shardTemplate, String suffix, int shardNum,
+      int numShards) {
+    return constructName(prefix, shardTemplate, suffix, shardNum, numShards, null, null);
+  }
+
   @Override
   @Nullable
   public ResourceId unwindowedFilename(ResourceId outputDirectory, Context context,
@@ -138,9 +209,57 @@ public ResourceId unwindowedFilename(ResourceId outputDirectory, Context context
 
   @Override
   public ResourceId windowedFilename(ResourceId outputDirectory,
-      WindowedContext c, String extension) {
-    throw new UnsupportedOperationException("There is no default policy for windowed file"
-        + " output. Please provide an explicit FilenamePolicy to generate filenames.");
+      WindowedContext context, String extension) {
+
+    boolean shardTemplateProvidedByUser = !this.shardTemplate.equals(DEFAULT_SHARD_TEMPLATE);
+
+    if (shardTemplateProvidedByUser){
+      boolean isWindowed = isWindowedTemplate(this.shardTemplate);
+      if (!isWindowed){
+        LOG.info("Template you provided {} does not have enough information to create"
+            + "meaningful windowed file names. Consider using P and W in your template",
+            this.shardTemplate);
+      }
+    }
+
+    final PaneInfo paneInfo = context.getPaneInfo();
+    String paneStr = paneInfoToString(paneInfo);
+    String windowStr = windowToString(context.getWindow());
+
+    String templateToUse = shardTemplate;
+    if (!shardTemplateProvidedByUser){
+      LOG.info("User did not provide shard template. For creating windowed file names "
+          + "default template {} will be used", DEFAULT_WINDOWED_SHARD_TEMPLATE);
+      templateToUse = DEFAULT_WINDOWED_SHARD_TEMPLATE;
+    }
+
+    String filename = constructName(prefix.get(), templateToUse, suffix,
+        context.getShardNumber(), context.getNumShards(), paneStr, windowStr)
+        + extension;
+    return outputDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
+  }
+
+  /*
+   * Since not all windows have toString() that is nice or is compatible to be a part of file name.
+   */
+  private String windowToString(BoundedWindow window) {
+    if (window instanceof GlobalWindow) {
+      return "GlobalWindow";
+    }
+    if (window instanceof IntervalWindow) {
+      IntervalWindow iw = (IntervalWindow) window;
+      return String.format("IntervalWindow-%s-%s", iw.start().toString(),
+          iw.end().toString());
+    }
+    return window.toString();
+  }
+
+  private String paneInfoToString(PaneInfo paneInfo){
+    long currentPaneIndex = (paneInfo == null ? -1L
+        : paneInfo.getIndex());
+    boolean firstPane = (paneInfo == null ? false : paneInfo.isFirst());
+    boolean lastPane = (paneInfo == null ? false : paneInfo.isLast());
+    return String.format("pane-%s-%b-%b", currentPaneIndex, firstPane, lastPane);
   }
 
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
index 5c068ce06441..afb5849ef7c0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
@@ -70,8 +70,10 @@
  * {@link TextIO.Write#withWindowedWrites()} will cause windowing and triggering to be
  * preserved. When producing windowed writes, the number of output shards must be set explicitly
  * using {@link TextIO.Write#withNumShards(int)}; some runners may set this for you to a
- * runner-chosen value, so you may need not set it yourself. A {@link FilenamePolicy} must be
- * set, and unique windows and triggers must produce unique filenames.
+ * runner-chosen value, so you may need not set it yourself. A {@link FilenamePolicy} can also be
+ * set in case you need better control over naming files created by unique windows.
+ * {@link DefaultFilenamePolicy} policy for producing unique filenames might not be appropriate
+ * for your use case.
  *
  * <p>Any existing files with the same names as generated output files will be overwritten.
  *
@@ -434,8 +436,6 @@ public PDone expand(PCollection<String> input) {
           (getFilenamePolicy() == null)
               || (getShardTemplate() == null && getFilenameSuffix() == null),
           "Cannot set a filename policy and also a filename template or suffix.");
-      checkState(!getWindowedWrites() || (getFilenamePolicy() != null),
-          "When using windowed writes, a filename policy must be set via withFilenamePolicy().");
 
       FilenamePolicy usedFilenamePolicy = getFilenamePolicy();
       if (usedFilenamePolicy == null) {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java
index c895da87f718..787403bad308 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java
@@ -18,7 +18,10 @@
 package org.apache.beam.sdk.io;
 
 import static org.apache.beam.sdk.io.DefaultFilenamePolicy.constructName;
+import static org.apache.beam.sdk.io.DefaultFilenamePolicy.isWindowedTemplate;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -29,6 +32,7 @@
  */
 @RunWith(JUnit4.class)
 public class DefaultFilenamePolicyTest {
+
   @Test
   public void testConstructName() {
     assertEquals("output-001-of-123.txt",
@@ -52,4 +56,57 @@ public void testConstructNameWithLargeShardCount() {
     assertEquals("out-100-of-5000.txt",
         constructName("out", "-SS-of-NN", ".txt", 100, 5000));
   }
+
+  @Test
+  public void testIsWindowedTemplate(){
+    assertTrue(isWindowedTemplate("-SSS-of-NNN-P-W"));
+    assertTrue(isWindowedTemplate("-SSS-of-NNN-W"));
+    assertTrue(isWindowedTemplate("-SSS-of-NNN-P"));
+    assertTrue(isWindowedTemplate("W-SSS-of-NNN"));
+
+    assertFalse(isWindowedTemplate("-SSS-of-NNN"));
+    assertFalse(isWindowedTemplate("-SSS-of-lp"));
+  }
+
+  @Test
+  public void testConstructWindowedName() {
+    assertEquals("output-001-of-123.txt",
+        constructName("output", "-SSS-of-NNN", ".txt", 1, 123, null, null));
+
+    assertEquals("output-001-of-123-PPP-W.txt",
+        constructName("output", "-SSS-of-NNN-PPP-W", ".txt", 1, 123, null, null));
+
+    assertEquals("out.txt/part-00042-myPaneStr-myWindowStr",
+        constructName("out.txt", "/part-SSSSS-P-W", "", 42, 100, "myPaneStr",
+            "myWindowStr"));
+
+    assertEquals("out.txt", constructName("ou", "t.t", "xt", 1, 1, "myPaneStr2",
+        "anotherWindowStr"));
+
+    assertEquals("out0102shard-oneMoreWindowStr-anotherPaneStr.txt",
+        constructName("out", "SSNNshard-W-P", ".txt", 1, 2, "anotherPaneStr",
+            "oneMoreWindowStr"));
+
+    assertEquals("out-2/1.part-1-of-2-slidingWindow1-myPaneStr3-windowslidingWindow1-"
+        + "panemyPaneStr3.txt",
+        constructName("out", "-N/S.part-S-of-N-W-P-windowW-paneP", ".txt", 1, 2, "myPaneStr3",
+        "slidingWindow1"));
+
+    // test first/last pane
+    assertEquals("out.txt/part-00042-myWindowStr-pane-11-true-false",
+        constructName("out.txt", "/part-SSSSS-W-P", "", 42, 100, "pane-11-true-false",
+            "myWindowStr"));
+
+    assertEquals("out.txt", constructName("ou", "t.t", "xt", 1, 1, "pane",
+        "anotherWindowStr"));
+
+    assertEquals("out0102shard-oneMoreWindowStr-pane--1-false-false-pane--1-false-false.txt",
+        constructName("out", "SSNNshard-W-P-P", ".txt", 1, 2, "pane--1-false-false",
+            "oneMoreWindowStr"));
+
+    assertEquals("out-2/1.part-1-of-2-sWindow1-winsWindow1-ppaneL.txt",
+        constructName("out",
+        "-N/S.part-S-of-N-W-winW-pP", ".txt", 1, 2, "paneL", "sWindow1"));
+  }
+
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
index 0d8fbbd34fa3..6c7a53f87664 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
@@ -1103,15 +1103,5 @@ public void testInitialSplitGzipModeGz() throws Exception {
     SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
   }
 
-  @Test
-  public void testWindowedWriteRequiresFilenamePolicy() {
-    PCollection<String> emptyInput = p.apply(Create.empty(StringUtf8Coder.of()));
-    TextIO.Write write = TextIO.write().to("/tmp/some/file").withWindowedWrites();
-
-    expectedException.expect(IllegalStateException.class);
-    expectedException.expectMessage(
-        "When using windowed writes, a filename policy must be set via withFilenamePolicy()");
-    emptyInput.apply(write);
-  }
 }
 

From de38410d3e2cf9c6edff9438d539929777ad7915 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=B3=A2=E7=89=B9?= <haozhi.shz@alibaba-inc.com>
Date: Thu, 20 Apr 2017 20:27:31 +0800
Subject: [PATCH 347/387] fix FlinkAccumulatorCombiningStateWithContext read
 null accum bug

---
 .../wrappers/streaming/state/FlinkStateInternals.java       | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
index 9cb742ee7e30..b73abe9b68b7 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
@@ -821,7 +821,11 @@ public OutputT read() {
                 flinkStateDescriptor);
 
         AccumT accum = state.value();
-        return combineFn.extractOutput(accum, context);
+        if (accum != null) {
+          return combineFn.extractOutput(accum, context);
+        } else {
+          return combineFn.extractOutput(combineFn.createAccumulator(context), context);
+        }
       } catch (Exception e) {
         throw new RuntimeException("Error reading state.", e);
       }

From bd1dfdf3c8e145a99bcacebd0c64dcf6580f3ffe Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 23 May 2017 13:29:51 -0700
Subject: [PATCH 348/387] Visit a Transform Hierarchy in Topological Order

Always ensure that the producer of a value is visited before that value
is visited for the first time. Visit a compoosite before visiting any of
its child nodes.
---
 .../beam/runners/spark/SparkRunner.java       | 13 +++
 .../spark/translation/BoundedDataset.java     |  6 ++
 .../translation/TransformTranslator.java      |  1 +
 .../spark/translation/StorageLevelTest.java   |  4 +-
 .../beam/sdk/runners/TransformHierarchy.java  | 46 +++++++++-
 .../sdk/runners/TransformHierarchyTest.java   | 86 +++++++++++++++++++
 6 files changed, 150 insertions(+), 6 deletions(-)

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 9e2426ef8381..1be5e13f4f5b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -21,8 +21,10 @@
 import com.google.common.collect.Iterables;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -49,6 +51,7 @@
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
@@ -336,6 +339,8 @@ public static class Evaluator extends Pipeline.PipelineVisitor.Defaults {
     protected final EvaluationContext ctxt;
     protected final SparkPipelineTranslator translator;
 
+    private final Set<Node> shouldIgnoreChildren = new HashSet<>();
+
     public Evaluator(SparkPipelineTranslator translator, EvaluationContext ctxt) {
       this.translator = translator;
       this.ctxt = ctxt;
@@ -351,6 +356,7 @@ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
           LOG.info("Entering directly-translatable composite transform: '{}'", node.getFullName());
           LOG.debug("Composite transform class: '{}'", transformClass);
           doVisitTransform(node);
+          shouldIgnoreChildren.add(node);
           return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
         }
       }
@@ -392,6 +398,13 @@ protected boolean shouldDefer(TransformHierarchy.Node node) {
 
     @Override
     public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+      Node parent = node.getEnclosingNode();
+      while (!parent.isRootNode()) {
+        if (shouldIgnoreChildren.contains(parent)) {
+          return;
+        }
+        parent = parent.getEnclosingNode();
+      }
       doVisitTransform(node);
     }
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
index 652c75305f0c..a746634008ef 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
@@ -19,6 +19,7 @@
 package org.apache.beam.runners.spark.translation;
 
 import com.google.common.base.Function;
+import com.google.common.base.Joiner;
 import com.google.common.collect.Iterables;
 import java.util.List;
 import javax.annotation.Nullable;
@@ -97,8 +98,13 @@ public WindowedValue<T> apply(byte[] bytes) {
     return windowedValues;
   }
 
+  int timesCached = 0;
   @Override
   public void cache(String storageLevel) {
+    System.out.printf(
+        "Persisting Dataset %s for RDD %s (id %s) at level %s. %s times before%n",
+        this, getRDD(), getRDD().toDebugString(), storageLevel, timesCached++);
+    System.out.println(Joiner.on("\n\t").join(new Throwable().getStackTrace()));
     // populate the rdd if needed
     getRDD().persist(StorageLevel.fromString(storageLevel));
   }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index 742ea83539e2..6ca12c9ca6cc 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -407,6 +407,7 @@ public void evaluate(Read.Bounded<T> transform, EvaluationContext context) {
         JavaRDD<WindowedValue<T>> input = new SourceRDD.Bounded<>(
             jsc.sc(), transform.getSource(), runtimeContext, stepName).toJavaRDD();
         // cache to avoid re-evaluation of the source by Spark's lazy DAG evaluation.
+        System.out.println("Evaluating Bounded Read " + transform);
         context.putDataset(transform, new BoundedDataset<>(input.cache()));
       }
 
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
index 8f2e681c9e91..8bd6dae98a74 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
@@ -58,12 +58,12 @@ public static void teardown() {
 
   @Test
   public void test() throws Exception {
-    PCollection<String> pCollection = pipeline.apply(Create.of("foo"));
+    PCollection<String> pCollection = pipeline.apply("CreateFoo", Create.of("foo"));
 
     // by default, the Spark runner doesn't cache the RDD if it accessed only one time.
     // So, to "force" the caching of the RDD, we have to call the RDD at least two time.
     // That's why we are using Count fn on the PCollection.
-    pCollection.apply(Count.<String>globally());
+    pCollection.apply("CountAll", Count.<String>globally());
 
     PCollection<String> output = pCollection.apply(new StorageLevelPTransform());
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 2f0e8efd7de8..630d24cf68f1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -202,10 +202,12 @@ Node getProducer(PValue produced) {
     return producers.get(produced);
   }
 
+  int traversed = 0;
   public Set<PValue> visit(PipelineVisitor visitor) {
     finishSpecifying();
     Set<PValue> visitedValues = new HashSet<>();
-    root.visit(visitor, visitedValues);
+    traversed++;
+    root.visit(visitor, visitedValues, new HashSet<Node>(), new HashSet<Node>());
     return visitedValues;
   }
 
@@ -462,7 +464,22 @@ public Map<TupleTag<?>, PValue> getOutputs() {
      * <p>Provides an ordered visit of the input values, the primitive transform (or child nodes for
      * composite transforms), then the output values.
      */
-    private void visit(PipelineVisitor visitor, Set<PValue> visitedValues) {
+    private void visit(
+        PipelineVisitor visitor,
+        Set<PValue> visitedValues,
+        Set<Node> visitedNodes,
+        Set<Node> passedComposites) {
+      if (getEnclosingNode() != null && !visitedNodes.contains(getEnclosingNode())) {
+        getEnclosingNode().visit(visitor, visitedValues, visitedNodes, passedComposites);
+      }
+      if (!visitedNodes.add(this)) {
+        LOG.debug("Not revisiting previously visited node {}", this);
+        return;
+      } else if (childNodeOf(passedComposites)) {
+        LOG.debug("Not revisiting Node {} which is a child of a previously passed composite", this);
+        return;
+      }
+
       if (!finishedSpecifying) {
         finishSpecifying();
       }
@@ -470,22 +487,31 @@ private void visit(PipelineVisitor visitor, Set<PValue> visitedValues) {
       if (!isRootNode()) {
         // Visit inputs.
         for (PValue inputValue : inputs.values()) {
+          Node valueProducer = getProducer(inputValue);
+          if (!visitedNodes.contains(valueProducer)) {
+            valueProducer.visit(visitor, visitedValues, visitedNodes, passedComposites);
+          }
           if (visitedValues.add(inputValue)) {
-            visitor.visitValue(inputValue, getProducer(inputValue));
+            LOG.debug("Visiting input value {}", inputValue);
+            visitor.visitValue(inputValue, valueProducer);
           }
         }
       }
 
       if (isCompositeNode()) {
+        LOG.debug("Visiting composite node {}", this);
         PipelineVisitor.CompositeBehavior recurse = visitor.enterCompositeTransform(this);
 
         if (recurse.equals(CompositeBehavior.ENTER_TRANSFORM)) {
           for (Node child : parts) {
-            child.visit(visitor, visitedValues);
+            child.visit(visitor, visitedValues, visitedNodes, passedComposites);
           }
+        } else {
+          passedComposites.add(this);
         }
         visitor.leaveCompositeTransform(this);
       } else {
+        LOG.debug("Visiting primitive node {}", this);
         visitor.visitPrimitiveTransform(this);
       }
 
@@ -494,12 +520,24 @@ private void visit(PipelineVisitor visitor, Set<PValue> visitedValues) {
         // Visit outputs.
         for (PValue pValue : outputs.values()) {
           if (visitedValues.add(pValue)) {
+            LOG.debug("Visiting output value {}", pValue);
             visitor.visitValue(pValue, this);
           }
         }
       }
     }
 
+    private boolean childNodeOf(Set<Node> nodes) {
+      if (isRootNode()) {
+        return false;
+      }
+      Node parent = this.getEnclosingNode();
+      while (!parent.isRootNode() && !nodes.contains(parent)) {
+        parent = parent.getEnclosingNode();
+      }
+      return nodes.contains(parent);
+    }
+
     /**
      * Finish specifying a transform.
      *
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
index 1197d1b04eb6..2fe2817f534c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
@@ -19,6 +19,7 @@
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertThat;
@@ -32,6 +33,8 @@
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.Defaults;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.io.Read;
@@ -492,4 +495,87 @@ public void visitPrimitiveTransform(Node node) {
     assertThat(visitedPrimitiveNodes, containsInAnyOrder(upstreamNode, replacementParNode));
     assertThat(visitedValues, Matchers.<PValue>containsInAnyOrder(upstream, output));
   }
+
+  @Test
+  public void visitIsTopologicallyOrdered() {
+    PCollection<String> one =
+        PCollection.<String>createPrimitiveOutputInternal(
+                pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)
+            .setCoder(StringUtf8Coder.of());
+    final PCollection<Integer> two =
+        PCollection.<Integer>createPrimitiveOutputInternal(
+                pipeline, WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
+            .setCoder(VarIntCoder.of());
+    final PDone done = PDone.in(pipeline);
+    final TupleTag<String> oneTag = new TupleTag<String>() {};
+    final TupleTag<Integer> twoTag = new TupleTag<Integer>() {};
+    final PCollectionTuple oneAndTwo = PCollectionTuple.of(oneTag, one).and(twoTag, two);
+
+    hierarchy.pushNode("consumes_both", one, new PTransform<PCollection<String>, PDone>() {
+      @Override
+      public PDone expand(PCollection<String> input) {
+        return done;
+      }
+
+      @Override
+      public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+        return Collections.<TupleTag<?>, PValue>singletonMap(twoTag, two);
+      }
+    });
+    hierarchy.setOutput(done);
+    hierarchy.popNode();
+
+    final PTransform<PBegin, PCollectionTuple> producer =
+        new PTransform<PBegin, PCollectionTuple>() {
+          @Override
+          public PCollectionTuple expand(PBegin input) {
+            return oneAndTwo;
+          }
+        };
+    hierarchy.pushNode(
+        "encloses_producer",
+        PBegin.in(pipeline),
+        new PTransform<PBegin, PCollectionTuple>() {
+          @Override
+          public PCollectionTuple expand(PBegin input) {
+            return input.apply(producer);
+          }
+        });
+    hierarchy.pushNode(
+        "creates_one_and_two",
+        PBegin.in(pipeline), producer);
+    hierarchy.setOutput(oneAndTwo);
+    hierarchy.popNode();
+    hierarchy.setOutput(oneAndTwo);
+    hierarchy.popNode();
+
+    hierarchy.visit(new PipelineVisitor.Defaults() {
+      private final Set<Node> visitedNodes = new HashSet<>();
+      private final Set<PValue> visitedValues = new HashSet<>();
+      @Override
+      public CompositeBehavior enterCompositeTransform(Node node) {
+        for (PValue input : node.getInputs().values()) {
+          assertThat(visitedValues, hasItem(input));
+        }
+        visitedNodes.add(node);
+        return CompositeBehavior.ENTER_TRANSFORM;
+      }
+
+      @Override
+      public void visitPrimitiveTransform(Node node) {
+        assertThat(visitedNodes, hasItem(node.getEnclosingNode()));
+        for (PValue input : node.getInputs().values()) {
+          assertThat(visitedValues, hasItem(input));
+        }
+        visitedNodes.add(node);
+      }
+
+      @Override
+      public void visitValue(PValue value, Node producer) {
+        assertThat(visitedNodes, hasItem(producer));
+        assertThat(visitedValues, not(hasItem(value)));
+        visitedValues.add(value);
+      }
+    });
+  }
 }

From 8eb09aad9c975f787ba8afac83394cc8b56eb94f Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 25 May 2017 10:41:56 -0700
Subject: [PATCH 349/387] Update Apex Overrides

Only override CreatePCollectionView transforms
---
 .../apache/beam/runners/apex/ApexRunner.java  | 119 ++++--------------
 1 file changed, 21 insertions(+), 98 deletions(-)

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index f91d8e550789..c595b3f50b2e 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -57,7 +57,6 @@
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.runners.PTransformOverrideFactory;
 import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -65,7 +64,7 @@
 import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.View.AsIterable;
-import org.apache.beam.sdk.transforms.View.AsSingleton;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PCollectionView;
@@ -111,16 +110,12 @@ private List<PTransformOverride> getOverrides() {
                 new PrimitiveCreate.Factory()))
         .add(
             PTransformOverride.of(
-                PTransformMatchers.classEqualTo(View.AsSingleton.class),
-                new StreamingViewAsSingleton.Factory()))
-        .add(
-            PTransformOverride.of(
-                PTransformMatchers.classEqualTo(View.AsIterable.class),
+                PTransformMatchers.createViewWithViewFn(PCollectionViews.IterableViewFn.class),
                 new StreamingViewAsIterable.Factory()))
         .add(
             PTransformOverride.of(
-                PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
-                new StreamingCombineGloballyAsSingletonView.Factory()))
+                PTransformMatchers.createViewWithViewFn(PCollectionViews.SingletonViewFn.class),
+                new StreamingWrapSingletonInList.Factory()))
         .add(
             PTransformOverride.of(
                 PTransformMatchers.splittableParDoMulti(),
@@ -245,117 +240,45 @@ public void processElement(ProcessContext c) {
     }
   }
 
-  private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
-      extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
+  private static class StreamingWrapSingletonInList<T>
+      extends PTransform<PCollection<T>, PCollectionView<T>> {
     private static final long serialVersionUID = 1L;
-    Combine.GloballyAsSingletonView<InputT, OutputT> transform;
+    CreatePCollectionView<T, T> transform;
 
     /**
      * Builds an instance of this class from the overridden transform.
      */
-    private StreamingCombineGloballyAsSingletonView(
-        Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
+    private StreamingWrapSingletonInList(
+        CreatePCollectionView<T, T> transform) {
       this.transform = transform;
     }
 
     @Override
-    public PCollectionView<OutputT> expand(PCollection<InputT> input) {
-      PCollection<OutputT> combined = input
-          .apply(Combine.globally(transform.getCombineFn())
-              .withoutDefaults().withFanout(transform.getFanout()));
-
-      PCollectionView<OutputT> view = PCollectionViews.singletonView(combined,
-          combined.getWindowingStrategy(), transform.getInsertDefault(),
-          transform.getInsertDefault() ? transform.getCombineFn().defaultValue() : null,
-              combined.getCoder());
-      return combined.apply(ParDo.of(new WrapAsList<OutputT>()))
-          .apply(CreateApexPCollectionView.<OutputT, OutputT> of(view));
+    public PCollectionView<T> expand(PCollection<T> input) {
+      return input
+          .apply(ParDo.of(new WrapAsList<T>()))
+          .apply(CreateApexPCollectionView.<T, T>of(transform.getView()));
     }
 
     @Override
     protected String getKindString() {
-      return "StreamingCombineGloballyAsSingletonView";
+      return "StreamingWrapSingletonInList";
     }
 
-    static class Factory<InputT, OutputT>
+    static class Factory<T>
         extends SingleInputOutputOverrideFactory<
-            PCollection<InputT>, PCollectionView<OutputT>,
-            Combine.GloballyAsSingletonView<InputT, OutputT>> {
+            PCollection<T>, PCollectionView<T>,
+            CreatePCollectionView<T, T>> {
       @Override
-      public PTransformReplacement<PCollection<InputT>, PCollectionView<OutputT>>
+      public PTransformReplacement<PCollection<T>, PCollectionView<T>>
           getReplacementTransform(
               AppliedPTransform<
-                      PCollection<InputT>, PCollectionView<OutputT>,
-                      GloballyAsSingletonView<InputT, OutputT>>
+                      PCollection<T>, PCollectionView<T>,
+                      CreatePCollectionView<T, T>>
                   transform) {
         return PTransformReplacement.of(
             PTransformReplacements.getSingletonMainInput(transform),
-            new StreamingCombineGloballyAsSingletonView<>(transform.getTransform()));
-      }
-    }
-  }
-
-  private static class StreamingViewAsSingleton<T>
-      extends PTransform<PCollection<T>, PCollectionView<T>> {
-    private static final long serialVersionUID = 1L;
-
-    private View.AsSingleton<T> transform;
-
-    public StreamingViewAsSingleton(View.AsSingleton<T> transform) {
-      this.transform = transform;
-    }
-
-    @Override
-    public PCollectionView<T> expand(PCollection<T> input) {
-      Combine.Globally<T, T> combine = Combine
-          .globally(new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
-      if (!transform.hasDefaultValue()) {
-        combine = combine.withoutDefaults();
-      }
-      return input.apply(combine.asSingletonView());
-    }
-
-    @Override
-    protected String getKindString() {
-      return "StreamingViewAsSingleton";
-    }
-
-    private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
-      private boolean hasDefaultValue;
-      private T defaultValue;
-
-      SingletonCombine(boolean hasDefaultValue, T defaultValue) {
-        this.hasDefaultValue = hasDefaultValue;
-        this.defaultValue = defaultValue;
-      }
-
-      @Override
-      public T apply(T left, T right) {
-        throw new IllegalArgumentException("PCollection with more than one element "
-            + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
-            + "combine the PCollection into a single value");
-      }
-
-      @Override
-      public T identity() {
-        if (hasDefaultValue) {
-          return defaultValue;
-        } else {
-          throw new IllegalArgumentException("Empty PCollection accessed as a singleton view. "
-              + "Consider setting withDefault to provide a default value");
-        }
-      }
-    }
-
-    static class Factory<T>
-        extends SingleInputOutputOverrideFactory<
-            PCollection<T>, PCollectionView<T>, View.AsSingleton<T>> {
-      @Override
-      public PTransformReplacement<PCollection<T>, PCollectionView<T>> getReplacementTransform(
-          AppliedPTransform<PCollection<T>, PCollectionView<T>, AsSingleton<T>> transform) {
-        return PTransformReplacement.of(
-            PTransformReplacements.getSingletonMainInput(transform),
-            new StreamingViewAsSingleton<>(transform.getTransform()));
+            new StreamingWrapSingletonInList<>(transform.getTransform()));
       }
     }
   }

From 2e2ae9cfa581a73864695d15102acadc2750a57a Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Tue, 23 May 2017 09:43:45 -0700
Subject: [PATCH 350/387] Roll-forward Include Additional PTransform inputs in
 Transform Nodes

Add the value of PTransform.getAdditionalInputs in the inputs of a
TransformHierarchy node.

Fork the Node constructor to reduce nullability

This slightly simplifies the constructor implementation(s).
---
 .../apex/translation/TranslationContext.java  |   4 +-
 .../core/construction/TransformInputs.java    |  50 ++++++
 .../construction/TransformInputsTest.java     | 166 ++++++++++++++++++
 .../runners/direct/DirectGraphVisitor.java    |  15 +-
 .../runners/direct/ParDoEvaluatorFactory.java |   9 +-
 ...ttableProcessElementsEvaluatorFactory.java |   2 +
 .../direct/StatefulParDoEvaluatorFactory.java |   1 +
 .../beam/runners/direct/WatermarkManager.java |  17 +-
 .../runners/direct/ParDoEvaluatorTest.java    |   6 +-
 .../flink/FlinkBatchTranslationContext.java   |   3 +-
 .../FlinkStreamingTranslationContext.java     |   3 +-
 .../dataflow/DataflowPipelineTranslator.java  |   5 +-
 .../spark/translation/EvaluationContext.java  |   4 +-
 .../beam/sdk/runners/TransformHierarchy.java  |  28 ++-
 14 files changed, 284 insertions(+), 29 deletions(-)
 create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
 create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index aff3863624c4..94d13e177dec 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -34,6 +34,7 @@
 import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -93,7 +94,8 @@ public Map<TupleTag<?>, PValue> getInputs() {
   }
 
   public <InputT extends PValue> InputT getInput() {
-    return (InputT) Iterables.getOnlyElement(getCurrentTransform().getInputs().values());
+    return (InputT)
+        Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
   }
 
   public Map<TupleTag<?>, PValue> getOutputs() {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
new file mode 100644
index 000000000000..2baf93a3c128
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.ImmutableList;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Utilities for extracting subsets of inputs from an {@link AppliedPTransform}. */
+public class TransformInputs {
+  /**
+   * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link
+   * PTransform#getAdditionalInputs()}.
+   */
+  public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) {
+    ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder();
+    PTransform<?, ?> transform = application.getTransform();
+    for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) {
+      if (!transform.getAdditionalInputs().containsKey(input.getKey())) {
+        mainInputs.add(input.getValue());
+      }
+    }
+    checkArgument(
+        !mainInputs.build().isEmpty() || application.getInputs().isEmpty(),
+        "Expected at least one main input if any inputs exist");
+    return mainInputs.build();
+  }
+}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
new file mode 100644
index 000000000000..f5b2c11e7923
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.junit.Assert.assertThat;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.hamcrest.Matchers;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link TransformInputs}. */
+@RunWith(JUnit4.class)
+public class TransformInputsTest {
+  @Rule public TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+  @Rule public ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  public void nonAdditionalInputsWithNoInputSucceeds() {
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "input-free",
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(),
+            pipeline);
+
+    assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>empty());
+  }
+
+  @Test
+  public void nonAdditionalInputsWithOneMainInputSucceeds() {
+    PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L));
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "input-single",
+            Collections.<TupleTag<?>, PValue>singletonMap(new TupleTag<Long>() {}, input),
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(),
+            pipeline);
+
+    assertThat(
+        TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>containsInAnyOrder(input));
+  }
+
+  @Test
+  public void nonAdditionalInputsWithMultipleNonAdditionalInputsSucceeds() {
+    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+    allInputs.put(new TupleTag<Integer>() {}, mainInts);
+    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+    allInputs.put(new TupleTag<Void>() {}, voids);
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "additional-free",
+            allInputs,
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(),
+            pipeline);
+
+    assertThat(
+        TransformInputs.nonAdditionalInputs(transform),
+        Matchers.<PValue>containsInAnyOrder(voids, mainInts));
+  }
+
+  @Test
+  public void nonAdditionalInputsWithAdditionalInputsSucceeds() {
+    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+    allInputs.put(new TupleTag<Integer>() {}, mainInts);
+    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+    allInputs.put(
+        new TupleTag<Void>() {}, voids);
+    allInputs.putAll(additionalInputs);
+
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "additional",
+            allInputs,
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(additionalInputs),
+            pipeline);
+
+    assertThat(
+        TransformInputs.nonAdditionalInputs(transform),
+        Matchers.<PValue>containsInAnyOrder(mainInts, voids));
+  }
+
+  @Test
+  public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() {
+    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+    AppliedPTransform<PInput, POutput, TestTransform> transform =
+        AppliedPTransform.of(
+            "additional-only",
+            additionalInputs,
+            Collections.<TupleTag<?>, PValue>emptyMap(),
+            new TestTransform(additionalInputs),
+            pipeline);
+
+    thrown.expect(IllegalArgumentException.class);
+    thrown.expectMessage("at least one");
+    TransformInputs.nonAdditionalInputs(transform);
+  }
+
+  private static class TestTransform extends PTransform<PInput, POutput> {
+    private final Map<TupleTag<?>, PValue> additionalInputs;
+
+    private TestTransform() {
+      this(Collections.<TupleTag<?>, PValue>emptyMap());
+    }
+
+    private TestTransform(Map<TupleTag<?>, PValue> additionalInputs) {
+      this.additionalInputs = additionalInputs;
+    }
+
+    @Override
+    public POutput expand(PInput input) {
+      return PDone.in(input.getPipeline());
+    }
+
+    @Override
+    public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+      return additionalInputs;
+    }
+  }
+}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index 01204e3049dd..ed4282bbdb8b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -21,10 +21,12 @@
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -34,6 +36,8 @@
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Tracks the {@link AppliedPTransform AppliedPTransforms} that consume each {@link PValue} in the
@@ -41,6 +45,7 @@
  * input after the upstream transform has produced and committed output.
  */
 class DirectGraphVisitor extends PipelineVisitor.Defaults {
+  private static final Logger LOG = LoggerFactory.getLogger(DirectGraphVisitor.class);
 
   private Map<POutput, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
 
@@ -83,7 +88,15 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
     if (node.getInputs().isEmpty()) {
       rootTransforms.add(appliedTransform);
     } else {
-      for (PValue value : node.getInputs().values()) {
+      Collection<PValue> mainInputs =
+          TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline()));
+      if (!mainInputs.containsAll(node.getInputs().values())) {
+        LOG.debug(
+            "Inputs reduced to {} from {} by removing additional inputs",
+            mainInputs,
+            node.getInputs().values());
+      }
+      for (PValue value : mainInputs) {
         primitiveConsumers.put(value, appliedTransform);
       }
     }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 8aa75cf1445e..516f798aba97 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -20,7 +20,6 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -79,6 +78,7 @@ public <T> TransformEvaluator<T> forApplication(
         (TransformEvaluator<T>)
             createEvaluator(
                 (AppliedPTransform) application,
+                (PCollection<InputT>) inputBundle.getPCollection(),
                 inputBundle.getKey(),
                 doFn,
                 transform.getSideInputs(),
@@ -102,6 +102,7 @@ public void cleanup() throws Exception {
   @SuppressWarnings({"unchecked", "rawtypes"})
   DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
+      PCollection<InputT> mainInput,
       StructuralKey<?> inputBundleKey,
       DoFn<InputT, OutputT> doFn,
       List<PCollectionView<?>> sideInputs,
@@ -120,6 +121,7 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
         createParDoEvaluator(
             application,
             inputBundleKey,
+            mainInput,
             sideInputs,
             mainOutputTag,
             additionalOutputTags,
@@ -132,6 +134,7 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
   ParDoEvaluator<InputT> createParDoEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
       StructuralKey<?> key,
+      PCollection<InputT> mainInput,
       List<PCollectionView<?>> sideInputs,
       TupleTag<OutputT> mainOutputTag,
       List<TupleTag<?>> additionalOutputTags,
@@ -144,8 +147,7 @@ ParDoEvaluator<InputT> createParDoEvaluator(
           evaluationContext,
           stepContext,
           application,
-          ((PCollection<InputT>) Iterables.getOnlyElement(application.getInputs().values()))
-              .getWindowingStrategy(),
+          mainInput.getWindowingStrategy(),
           fn,
           key,
           sideInputs,
@@ -173,5 +175,4 @@ static Map<TupleTag<?>, PCollection<?>> pcollections(Map<TupleTag<?>, PValue> ou
     }
     return pcs;
   }
-
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index b85f481c1489..eccc83a031cb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -116,6 +116,8 @@ public void cleanup() throws Exception {
             delegateFactory.createParDoEvaluator(
                 application,
                 inputBundle.getKey(),
+                (PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>)
+                    inputBundle.getPCollection(),
                 transform.getSideInputs(),
                 transform.getMainOutputTag(),
                 transform.getAdditionalOutputTags().getAll(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 506c84cec639..3619d05b47ca 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -117,6 +117,7 @@ private TransformEvaluator<KeyedWorkItem<K, KV<K, InputT>>> createEvaluator(
     DoFnLifecycleManagerRemovingTransformEvaluator<KV<K, InputT>> delegateEvaluator =
         delegateFactory.createEvaluator(
             (AppliedPTransform) application,
+            (PCollection) inputBundle.getPCollection(),
             inputBundle.getKey(),
             doFn,
             application.getTransform().getUnderlyingParDo().getSideInputs(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index 4f1b8319dc2d..b15b52e314de 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -823,10 +823,11 @@ private Collection<Watermark> getInputProcessingWatermarks(AppliedPTransform<?,
       inputWmsBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      Watermark producerOutputWatermark =
-          getTransformWatermark(graph.getProducer(pvalue))
-              .synchronizedProcessingOutputWatermark;
-      inputWmsBuilder.add(producerOutputWatermark);
+      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
+        Watermark producerOutputWatermark =
+            getTransformWatermark(graph.getProducer(pvalue)).synchronizedProcessingOutputWatermark;
+        inputWmsBuilder.add(producerOutputWatermark);
+      }
     }
     return inputWmsBuilder.build();
   }
@@ -838,9 +839,11 @@ private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform)
       inputWatermarksBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      Watermark producerOutputWatermark =
-          getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
-      inputWatermarksBuilder.add(producerOutputWatermark);
+      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
+        Watermark producerOutputWatermark =
+            getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
+        inputWatermarksBuilder.add(producerOutputWatermark);
+      }
     }
     List<Watermark> inputCollectionWatermarks = inputWatermarksBuilder.build();
     return inputCollectionWatermarks;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 09a21ac524a5..8b86bbe50958 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -98,7 +98,7 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
     when(evaluationContext.createBundle(output)).thenReturn(outputBundle);
 
     ParDoEvaluator<Integer> evaluator =
-        createEvaluator(singletonView, fn, output);
+        createEvaluator(singletonView, fn, inputPc, output);
 
     IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
     WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
@@ -132,6 +132,7 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
   private ParDoEvaluator<Integer> createEvaluator(
       PCollectionView<Integer> singletonView,
       RecorderFn fn,
+      PCollection<Integer> input,
       PCollection<Integer> output) {
     when(
             evaluationContext.createSideInputReader(
@@ -156,8 +157,7 @@ private ParDoEvaluator<Integer> createEvaluator(
         evaluationContext,
         stepContext,
         transform,
-        ((PCollection<?>) Iterables.getOnlyElement(transform.getInputs().values()))
-            .getWindowingStrategy(),
+        input.getWindowingStrategy(),
         fn,
         null /* key */,
         ImmutableList.<PCollectionView<?>>of(singletonView),
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index 0439119dfc40..6e7019848b19 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -20,6 +20,7 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -143,7 +144,7 @@ Map<TupleTag<?>, PValue> getInputs(PTransform<?, ?> transform) {
 
   @SuppressWarnings("unchecked")
   <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
   }
 
   Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
index ea5f6b3162af..74a5fb971144 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
@@ -22,6 +22,7 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -113,7 +114,7 @@ public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collecti
 
   @SuppressWarnings("unchecked")
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
   }
 
   public <T extends PInput> Map<TupleTag<?>, PValue> getInputs(PTransform<T, ?> transform) {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index af93ef59bba4..fccd018b13d2 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -56,6 +56,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.core.construction.WindowingStrategyTranslation;
 import org.apache.beam.runners.dataflow.BatchViewOverrides.GroupByKeyAndSortValuesOnly;
 import org.apache.beam.runners.dataflow.DataflowRunner.CombineGroupedValues;
@@ -395,7 +396,9 @@ public <InputT extends PInput> Map<TupleTag<?>, PValue> getInputs(
 
     @Override
     public <InputT extends PValue> InputT getInput(PTransform<InputT, ?> transform) {
-      return (InputT) Iterables.getOnlyElement(getInputs(transform).values());
+      return (InputT)
+          Iterables.getOnlyElement(
+              TransformInputs.nonAdditionalInputs(getCurrentTransform(transform)));
     }
 
     @Override
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 8102926f6daa..0c6c4d1cb660 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -26,6 +26,7 @@
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
+import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.sdk.Pipeline;
@@ -103,7 +104,8 @@ public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
 
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
     @SuppressWarnings("unchecked")
-    T input = (T) Iterables.getOnlyElement(getInputs(transform).values());
+    T input =
+        (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
     return input;
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 630d24cf68f1..bebc306a8d24 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -32,7 +32,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
@@ -68,7 +67,7 @@ public TransformHierarchy() {
     producers = new HashMap<>();
     producerInput = new HashMap<>();
     unexpandedInputs = new HashMap<>();
-    root = new Node(null, null, "", null);
+    root = new Node();
     current = root;
   }
 
@@ -254,26 +253,37 @@ public class Node {
     @VisibleForTesting
     boolean finishedSpecifying = false;
 
+    /**
+     * Creates the root-level node. The root level node has a null enclosing node, a null transform,
+     * an empty map of inputs, and a name equal to the empty string.
+     */
+    private Node() {
+      this.enclosingNode = null;
+      this.transform = null;
+      this.fullName = "";
+      this.inputs = Collections.emptyMap();
+    }
+
     /**
      * Creates a new Node with the given parent and transform.
      *
-     * <p>EnclosingNode and transform may both be null for a root-level node, which holds all other
-     * nodes.
-     *
      * @param enclosingNode the composite node containing this node
      * @param transform the PTransform tracked by this node
      * @param fullName the fully qualified name of the transform
      * @param input the unexpanded input to the transform
      */
     private Node(
-        @Nullable Node enclosingNode,
-        @Nullable PTransform<?, ?> transform,
+        Node enclosingNode,
+        PTransform<?, ?> transform,
         String fullName,
-        @Nullable PInput input) {
+        PInput input) {
       this.enclosingNode = enclosingNode;
       this.transform = transform;
       this.fullName = fullName;
-      this.inputs = input == null ? Collections.<TupleTag<?>, PValue>emptyMap() : input.expand();
+      ImmutableMap.Builder<TupleTag<?>, PValue> inputs = ImmutableMap.builder();
+      inputs.putAll(input.expand());
+      inputs.putAll(transform.getAdditionalInputs());
+      this.inputs = inputs.build();
     }
 
     /**

From c8478fe1fe107b842d3cfa56b652d740fdf0c18b Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 25 May 2017 07:25:08 -0700
Subject: [PATCH 351/387] Mark CombineFnWithContext StateSpecs internal

---
 .../java/org/apache/beam/sdk/state/StateSpecs.java   | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
index 7b7138489997..5a2a1b6b9c72 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
@@ -76,7 +76,9 @@ StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
   }
 
   /**
-   * Create a {@link StateSpec} for a {@link CombiningState} which uses a {@link
+   * <b>For internal use only; no backwards compatibility guarantees</b>
+   *
+   * <p>Create a {@link StateSpec} for a {@link CombiningState} which uses a {@link
    * CombineFnWithContext} to automatically merge multiple values of type {@code InputT} into a
    * single resulting {@code OutputT}.
    *
@@ -84,6 +86,7 @@ StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
    *
    * @see #combining(Coder, CombineFnWithContext)
    */
+  @Internal
   public static <InputT, AccumT, OutputT>
       StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
           CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {
@@ -105,11 +108,14 @@ StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
   }
 
   /**
-   * Identical to {@link #combining(CombineFnWithContext)}, but with an accumulator coder explicitly
-   * supplied.
+   * <b>For internal use only; no backwards compatibility guarantees</b>
+   *
+   * <p>Identical to {@link #combining(CombineFnWithContext)}, but with an accumulator coder
+   * explicitly supplied.
    *
    * <p>If automatic coder inference fails, use this method.
    */
+  @Internal
   public static <InputT, AccumT, OutputT>
       StateSpec<CombiningState<InputT, AccumT, OutputT>> combining(
           Coder<AccumT> accumCoder, CombineFnWithContext<InputT, AccumT, OutputT> combineFn) {

From b0dc523c72a68e870392fbac8ff9f3a87459ab22 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 25 May 2017 13:02:15 -0700
Subject: [PATCH 352/387] Allow translation to throw IOException

---
 .../beam/runners/core/construction/PTransformTranslation.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 9f5f3b50b723..00ea55e6e1ed 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -138,7 +138,8 @@ public static String urnForTransform(PTransform<?, ?> transform) {
    */
   public interface TransformPayloadTranslator<T extends PTransform<?, ?>> {
     String getUrn(T transform);
-    FunctionSpec translate(AppliedPTransform<?, ?, T> application, SdkComponents components);
+    FunctionSpec translate(AppliedPTransform<?, ?, T> application, SdkComponents components)
+        throws IOException;
   }
 
   /**

From 9497e5eaecf5d7eb7f18709935c183b03116f75f Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 25 May 2017 07:12:08 -0700
Subject: [PATCH 353/387] Flesh out TimerSpec and StateSpec in Runner API

---
 .../src/main/proto/beam_runner_api.proto      | 40 ++++++++++++++-----
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
index c8722e6a39fb..16122093c812 100644
--- a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
+++ b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto
@@ -247,21 +247,39 @@ message Parameter {
 }
 
 message StateSpec {
-  // TODO: AST for state spec
-  string id = 1;
-  Type type = 2;
-
-  enum Type {
-    VALUE = 0;
-    BAG = 1;
-    MAP = 2;
-    SET = 3;
+  oneof spec {
+    ValueStateSpec value_spec = 1;
+    BagStateSpec bag_spec = 2;
+    CombiningStateSpec combining_spec = 3;
+    MapStateSpec map_spec = 4;
+    SetStateSpec set_spec = 5;
   }
 }
 
+message ValueStateSpec {
+  string coder_id = 1;
+}
+
+message BagStateSpec {
+  string element_coder_id = 1;
+}
+
+message CombiningStateSpec {
+  string accumulator_coder_id = 1;
+  SdkFunctionSpec combine_fn = 2;
+}
+
+message MapStateSpec {
+  string key_coder_id = 1;
+  string value_coder_id = 2;
+}
+
+message SetStateSpec {
+  string element_coder_id = 1;
+}
+
 message TimerSpec {
-  // TODO: AST for timer spec
-  string id = 1;
+  TimeDomain time_domain = 1;
 }
 
 enum IsBounded {

From 8fc2eb0aeee9c3bdeaf93897e5e8aa4bb98b98de Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 25 May 2017 07:27:52 -0700
Subject: [PATCH 354/387] Add case dispatch to StateSpec

This is different than a StateBinder: for a binder, the id is needed and
the StateSpec controls the return type. For case dispatch, the
dispatcher controls the type and it should just be reading the spec,
which does not require the id. Eventually, StateBinder could be removed
in favor of StateSpec.Cases<Function<String, StateT>>.
---
 .../org/apache/beam/sdk/state/StateSpec.java  | 53 +++++++++++++++++++
 .../org/apache/beam/sdk/state/StateSpecs.java | 41 ++++++++++++++
 2 files changed, 94 insertions(+)

diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
index b0412bf49c6d..0443f25f7c4e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpec.java
@@ -22,6 +22,7 @@
 import org.apache.beam.sdk.annotations.Experimental.Kind;
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.Combine;
 
 /**
  * A specification of a persistent state cell. This includes information necessary to encode the
@@ -40,6 +41,14 @@ public interface StateSpec<StateT extends State> extends Serializable {
   @Internal
   StateT bind(String id, StateBinder binder);
 
+  /**
+   * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
+   *
+   * <p>Perform case analysis on this {@link StateSpec} using the provided {@link Cases}.
+   */
+  @Internal
+  <ResultT> ResultT match(Cases<ResultT> cases);
+
   /**
    * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
    *
@@ -60,4 +69,48 @@ public interface StateSpec<StateT extends State> extends Serializable {
    */
   @Internal
   void finishSpecifying();
+
+  /**
+   * Cases for doing a "switch" on the type of {@link StateSpec}.
+   */
+  interface Cases<ResultT> {
+    ResultT dispatchValue(Coder<?> valueCoder);
+    ResultT dispatchBag(Coder<?> elementCoder);
+    ResultT dispatchCombining(Combine.CombineFn<?, ?, ?> combineFn, Coder<?> accumCoder);
+    ResultT dispatchMap(Coder<?> keyCoder, Coder<?> valueCoder);
+    ResultT dispatchSet(Coder<?> elementCoder);
+
+    /**
+     * A base class for a visitor with a default method for cases it is not interested in.
+     */
+    abstract class WithDefault<ResultT> implements Cases<ResultT> {
+
+      protected abstract ResultT dispatchDefault();
+
+      @Override
+      public ResultT dispatchValue(Coder<?> valueCoder) {
+        return dispatchDefault();
+      }
+
+      @Override
+      public ResultT dispatchBag(Coder<?> elementCoder) {
+        return dispatchDefault();
+      }
+
+      @Override
+      public ResultT dispatchCombining(Combine.CombineFn<?, ?, ?> combineFn, Coder<?> accumCoder) {
+        return dispatchDefault();
+      }
+
+      @Override
+      public ResultT dispatchMap(Coder<?> keyCoder, Coder<?> valueCoder) {
+        return dispatchDefault();
+      }
+
+      @Override
+      public ResultT dispatchSet(Coder<?> elementCoder) {
+        return dispatchDefault();
+      }
+    }
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
index 5a2a1b6b9c72..42223047cc58 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java
@@ -278,6 +278,11 @@ public ValueState<T> bind(String id, StateBinder visitor) {
       return visitor.bindValue(id, this, coder);
     }
 
+    @Override
+    public <ResultT> ResultT match(Cases<ResultT> cases) {
+      return cases.dispatchValue(coder);
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public void offerCoders(Coder[] coders) {
@@ -342,6 +347,11 @@ public CombiningState<InputT, AccumT, OutputT> bind(
       return visitor.bindCombining(id, this, accumCoder, combineFn);
     }
 
+    @Override
+    public <ResultT> ResultT match(Cases<ResultT> cases) {
+      return cases.dispatchCombining(combineFn, accumCoder);
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public void offerCoders(Coder[] coders) {
@@ -413,6 +423,14 @@ public CombiningState<InputT, AccumT, OutputT> bind(
       return visitor.bindCombiningWithContext(id, this, accumCoder, combineFn);
     }
 
+    @Override
+    public <ResultT> ResultT match(Cases<ResultT> cases) {
+      throw new UnsupportedOperationException(
+          String.format(
+              "%s is for internal use only and does not support case dispatch",
+              getClass().getSimpleName()));
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public void offerCoders(Coder[] coders) {
@@ -480,6 +498,11 @@ public BagState<T> bind(String id, StateBinder visitor) {
       return visitor.bindBag(id, this, elemCoder);
     }
 
+    @Override
+    public <ResultT> ResultT match(Cases<ResultT> cases) {
+      return cases.dispatchBag(elemCoder);
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public void offerCoders(Coder[] coders) {
@@ -536,6 +559,11 @@ public MapState<K, V> bind(String id, StateBinder visitor) {
       return visitor.bindMap(id, this, keyCoder, valueCoder);
     }
 
+    @Override
+    public <ResultT> ResultT match(Cases<ResultT> cases) {
+      return cases.dispatchMap(keyCoder, valueCoder);
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public void offerCoders(Coder[] coders) {
@@ -600,6 +628,11 @@ public SetState<T> bind(String id, StateBinder visitor) {
       return visitor.bindSet(id, this, elemCoder);
     }
 
+    @Override
+    public <ResultT> ResultT match(Cases<ResultT> cases) {
+      return cases.dispatchSet(elemCoder);
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public void offerCoders(Coder[] coders) {
@@ -663,6 +696,14 @@ public WatermarkHoldState bind(String id, StateBinder visitor) {
       return visitor.bindWatermark(id, this, timestampCombiner);
     }
 
+    @Override
+    public <ResultT> ResultT match(Cases<ResultT> cases) {
+      throw new UnsupportedOperationException(
+          String.format(
+              "%s is for internal use only and does not support case dispatch",
+              getClass().getSimpleName()));
+    }
+
     @Override
     public void offerCoders(Coder[] coders) {
     }

From a250ce58c6a0caf473842c4e5e6f980a828dde55 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 25 May 2017 22:51:18 -0700
Subject: [PATCH 355/387] Make Java serialized CombineFn URN public

---
 .../beam/runners/core/construction/CombineTranslation.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
index 855fba740a02..28bc9a15e47f 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
@@ -49,7 +49,7 @@
  * RunnerApi.CombinePayload} protos.
  */
 public class CombineTranslation {
-  private static final String JAVA_SERIALIZED_COMBINE_FN_URN = "urn:beam:java:combinefn:v1";
+  public static final String JAVA_SERIALIZED_COMBINE_FN_URN = "urn:beam:java:combinefn:v1";
 
   public static CombinePayload toProto(
       AppliedPTransform<?, ?, Combine.PerKey<?, ?, ?>> combine, SdkComponents sdkComponents)

From 39220dbca944a2496587c543de2a4eb01004bd76 Mon Sep 17 00:00:00 2001
From: Kenneth Knowles <klk@google.com>
Date: Thu, 25 May 2017 07:12:29 -0700
Subject: [PATCH 356/387] Implement TimerSpec and StateSpec translation

---
 .../core/construction/CombineTranslation.java |   2 +-
 .../core/construction/ParDoTranslation.java   | 215 +++++++++++++++--
 .../construction/ParDoTranslationTest.java    | 218 +++++++++++-------
 3 files changed, 343 insertions(+), 92 deletions(-)

diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
index 28bc9a15e47f..472b6f8b1d48 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java
@@ -86,7 +86,7 @@ private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
         .getAccumulatorCoder();
   }
 
-  private static SdkFunctionSpec toProto(GlobalCombineFn<?, ?, ?> combineFn) {
+  public static SdkFunctionSpec toProto(GlobalCombineFn<?, ?, ?> combineFn) {
     return SdkFunctionSpec.newBuilder()
         // TODO: Set Java SDK Environment URN
         .setSpec(
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
index 1c81f8ce05d5..fe66179bc5e7 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java
@@ -19,10 +19,12 @@
 package org.apache.beam.runners.core.construction;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 import static org.apache.beam.runners.core.construction.PTransformTranslation.PAR_DO_TRANSFORM_URN;
 
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
@@ -46,9 +48,12 @@
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.SdkFunctionSpec;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput.Builder;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.StateSpec;
-import org.apache.beam.sdk.common.runner.v1.RunnerApi.TimerSpec;
 import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.state.StateSpec;
+import org.apache.beam.sdk.state.StateSpecs;
+import org.apache.beam.sdk.state.TimeDomain;
+import org.apache.beam.sdk.state.TimerSpec;
+import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Materializations;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -107,7 +112,8 @@ public String getUrn(ParDo.MultiOutput<?, ?> transform) {
 
     @Override
     public FunctionSpec translate(
-        AppliedPTransform<?, ?, MultiOutput<?, ?>> transform, SdkComponents components) {
+        AppliedPTransform<?, ?, MultiOutput<?, ?>> transform, SdkComponents components)
+        throws IOException {
       ParDoPayload payload = toProto(transform.getTransform(), components);
       return RunnerApi.FunctionSpec.newBuilder()
           .setUrn(PAR_DO_TRANSFORM_URN)
@@ -128,8 +134,10 @@ public static class Registrar implements TransformPayloadTranslatorRegistrar {
     }
   }
 
-  public static ParDoPayload toProto(ParDo.MultiOutput<?, ?> parDo, SdkComponents components) {
-    DoFnSignature signature = DoFnSignatures.getSignature(parDo.getFn().getClass());
+  public static ParDoPayload toProto(ParDo.MultiOutput<?, ?> parDo, SdkComponents components)
+  throws IOException {
+    DoFn<?, ?> doFn = parDo.getFn();
+    DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass());
     Map<String, StateDeclaration> states = signature.stateDeclarations();
     Map<String, TimerDeclaration> timers = signature.timerDeclarations();
     List<Parameter> parameters = signature.processElement().extraParameters();
@@ -146,16 +154,62 @@ public static ParDoPayload toProto(ParDo.MultiOutput<?, ?> parDo, SdkComponents
       }
     }
     for (Map.Entry<String, StateDeclaration> state : states.entrySet()) {
-      StateSpec spec = toProto(state.getValue());
+      RunnerApi.StateSpec spec =
+          toProto(getStateSpecOrCrash(state.getValue(), doFn), components);
       builder.putStateSpecs(state.getKey(), spec);
     }
     for (Map.Entry<String, TimerDeclaration> timer : timers.entrySet()) {
-      TimerSpec spec = toProto(timer.getValue());
+      RunnerApi.TimerSpec spec =
+          toProto(getTimerSpecOrCrash(timer.getValue(), doFn));
       builder.putTimerSpecs(timer.getKey(), spec);
     }
     return builder.build();
   }
 
+  private static StateSpec<?> getStateSpecOrCrash(
+      StateDeclaration stateDeclaration, DoFn<?, ?> target) {
+    try {
+      Object fieldValue = stateDeclaration.field().get(target);
+      checkState(fieldValue instanceof StateSpec,
+          "Malformed %s class %s: state declaration field %s does not have type %s.",
+          DoFn.class.getSimpleName(),
+          target.getClass().getName(),
+          stateDeclaration.field().getName(),
+          StateSpec.class);
+
+      return (StateSpec<?>) stateDeclaration.field().get(target);
+    } catch (IllegalAccessException exc) {
+      throw new RuntimeException(
+          String.format(
+              "Malformed %s class %s: state declaration field %s is not accessible.",
+              DoFn.class.getSimpleName(),
+              target.getClass().getName(),
+              stateDeclaration.field().getName()));
+    }
+  }
+
+  private static TimerSpec getTimerSpecOrCrash(
+      TimerDeclaration timerDeclaration, DoFn<?, ?> target) {
+    try {
+      Object fieldValue = timerDeclaration.field().get(target);
+      checkState(fieldValue instanceof TimerSpec,
+          "Malformed %s class %s: timer declaration field %s does not have type %s.",
+          DoFn.class.getSimpleName(),
+          target.getClass().getName(),
+          timerDeclaration.field().getName(),
+          TimerSpec.class);
+
+      return (TimerSpec) timerDeclaration.field().get(target);
+    } catch (IllegalAccessException exc) {
+      throw new RuntimeException(
+          String.format(
+              "Malformed %s class %s: timer declaration field %s is not accessible.",
+              DoFn.class.getSimpleName(),
+              target.getClass().getName(),
+              timerDeclaration.field().getName()));
+    }
+  }
+
   public static DoFn<?, ?> getDoFn(ParDoPayload payload) throws InvalidProtocolBufferException {
     return doFnAndMainOutputTagFromProto(payload.getDoFn()).getDoFn();
   }
@@ -179,14 +233,149 @@ public static RunnerApi.PCollection getMainInput(
     return components.getPcollectionsOrThrow(ptransform.getInputsOrThrow(mainInputId));
   }
 
-  // TODO: Implement
-  private static StateSpec toProto(StateDeclaration state) {
-    throw new UnsupportedOperationException("Not yet supported");
+  @VisibleForTesting
+  static RunnerApi.StateSpec toProto(StateSpec<?> stateSpec, final SdkComponents components)
+      throws IOException {
+    final RunnerApi.StateSpec.Builder builder = RunnerApi.StateSpec.newBuilder();
+
+    return stateSpec.match(
+        new StateSpec.Cases<RunnerApi.StateSpec>() {
+          @Override
+          public RunnerApi.StateSpec dispatchValue(Coder<?> valueCoder) {
+            return builder
+                .setValueSpec(
+                    RunnerApi.ValueStateSpec.newBuilder()
+                        .setCoderId(registerCoderOrThrow(components, valueCoder)))
+                .build();
+          }
+
+          @Override
+          public RunnerApi.StateSpec dispatchBag(Coder<?> elementCoder) {
+            return builder
+                .setBagSpec(
+                    RunnerApi.BagStateSpec.newBuilder()
+                        .setElementCoderId(registerCoderOrThrow(components, elementCoder)))
+                .build();
+          }
+
+          @Override
+          public RunnerApi.StateSpec dispatchCombining(
+              Combine.CombineFn<?, ?, ?> combineFn, Coder<?> accumCoder) {
+            return builder
+                .setCombiningSpec(
+                    RunnerApi.CombiningStateSpec.newBuilder()
+                        .setAccumulatorCoderId(registerCoderOrThrow(components, accumCoder))
+                        .setCombineFn(CombineTranslation.toProto(combineFn)))
+                .build();
+          }
+
+          @Override
+          public RunnerApi.StateSpec dispatchMap(Coder<?> keyCoder, Coder<?> valueCoder) {
+            return builder
+                .setMapSpec(
+                    RunnerApi.MapStateSpec.newBuilder()
+                        .setKeyCoderId(registerCoderOrThrow(components, keyCoder))
+                        .setValueCoderId(registerCoderOrThrow(components, valueCoder)))
+                .build();
+          }
+
+          @Override
+          public RunnerApi.StateSpec dispatchSet(Coder<?> elementCoder) {
+            return builder
+                .setSetSpec(
+                    RunnerApi.SetStateSpec.newBuilder()
+                        .setElementCoderId(registerCoderOrThrow(components, elementCoder)))
+                .build();
+          }
+        });
+  }
+
+  @VisibleForTesting
+  static StateSpec<?> fromProto(RunnerApi.StateSpec stateSpec, RunnerApi.Components components)
+      throws IOException {
+    switch (stateSpec.getSpecCase()) {
+      case VALUE_SPEC:
+        return StateSpecs.value(
+            CoderTranslation.fromProto(
+                components.getCodersMap().get(stateSpec.getValueSpec().getCoderId()), components));
+      case BAG_SPEC:
+        return StateSpecs.bag(
+            CoderTranslation.fromProto(
+                components.getCodersMap().get(stateSpec.getBagSpec().getElementCoderId()),
+                components));
+      case COMBINING_SPEC:
+        FunctionSpec combineFnSpec = stateSpec.getCombiningSpec().getCombineFn().getSpec();
+
+        if (!combineFnSpec.getUrn().equals(CombineTranslation.JAVA_SERIALIZED_COMBINE_FN_URN)) {
+          throw new UnsupportedOperationException(
+              String.format(
+                  "Cannot create %s from non-Java %s: %s",
+                  StateSpec.class.getSimpleName(),
+                  Combine.CombineFn.class.getSimpleName(),
+                  combineFnSpec.getUrn()));
+        }
+
+        Combine.CombineFn<?, ?, ?> combineFn =
+            (Combine.CombineFn<?, ?, ?>)
+                SerializableUtils.deserializeFromByteArray(
+                    combineFnSpec.getParameter().unpack(BytesValue.class).toByteArray(),
+                    Combine.CombineFn.class.getSimpleName());
+
+        // Rawtype coder cast because it is required to be a valid accumulator coder
+        // for the CombineFn, by construction
+        return StateSpecs.combining(
+            (Coder)
+                CoderTranslation.fromProto(
+                    components
+                        .getCodersMap()
+                        .get(stateSpec.getCombiningSpec().getAccumulatorCoderId()),
+                    components),
+            combineFn);
+
+      case MAP_SPEC:
+        return StateSpecs.map(
+            CoderTranslation.fromProto(
+                components.getCodersOrThrow(stateSpec.getMapSpec().getKeyCoderId()), components),
+            CoderTranslation.fromProto(
+                components.getCodersOrThrow(stateSpec.getMapSpec().getValueCoderId()), components));
+
+      case SET_SPEC:
+        return StateSpecs.set(
+            CoderTranslation.fromProto(
+                components.getCodersMap().get(stateSpec.getSetSpec().getElementCoderId()),
+                components));
+
+      case SPEC_NOT_SET:
+      default:
+        throw new IllegalArgumentException(
+            String.format("Unknown %s: %s", RunnerApi.StateSpec.class.getName(), stateSpec));
+
+    }
+  }
+
+  private static String registerCoderOrThrow(SdkComponents components, Coder coder) {
+    try {
+      return components.registerCoder(coder);
+    } catch (IOException exc) {
+      throw new RuntimeException("Failure to register coder", exc);
+    }
   }
 
-  // TODO: Implement
-  private static TimerSpec toProto(TimerDeclaration timer) {
-    throw new UnsupportedOperationException("Not yet supported");
+  private static RunnerApi.TimerSpec toProto(TimerSpec timer) {
+    return RunnerApi.TimerSpec.newBuilder().setTimeDomain(toProto(timer.getTimeDomain())).build();
+  }
+
+  private static RunnerApi.TimeDomain toProto(TimeDomain timeDomain) {
+    switch(timeDomain) {
+      case EVENT_TIME:
+        return RunnerApi.TimeDomain.EVENT_TIME;
+      case PROCESSING_TIME:
+        return RunnerApi.TimeDomain.PROCESSING_TIME;
+      case SYNCHRONIZED_PROCESSING_TIME:
+        return RunnerApi.TimeDomain.SYNCHRONIZED_PROCESSING_TIME;
+      default:
+        throw new IllegalArgumentException("Unknown time domain");
+    }
   }
 
   @AutoValue
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
index ec2795746451..46f6a806f292 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java
@@ -28,6 +28,7 @@
 import java.util.Map;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi;
 import org.apache.beam.sdk.common.runner.v1.RunnerApi.Components;
@@ -62,98 +63,159 @@
 import org.hamcrest.Matchers;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameter;
 import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Suite;
 
 /** Tests for {@link ParDoTranslation}. */
-@RunWith(Parameterized.class)
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+  ParDoTranslationTest.TestParDoPayloadTranslation.class,
+  ParDoTranslationTest.TestStateAndTimerTranslation.class
+})
 public class ParDoTranslationTest {
-  public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
-
-  private static PCollectionView<Long> singletonSideInput =
-      p.apply("GenerateSingleton", GenerateSequence.from(0L).to(1L))
-          .apply(View.<Long>asSingleton());
-  private static PCollectionView<Map<Long, Iterable<String>>> multimapSideInput =
-      p.apply("CreateMultimap", Create.of(KV.of(1L, "foo"), KV.of(1L, "bar"), KV.of(2L, "spam")))
-          .setCoder(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of()))
-          .apply(View.<Long, String>asMultimap());
-
-  private static PCollection<KV<Long, String>> mainInput =
-      p.apply("CreateMainInput", Create.empty(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of())));
-
-  @Parameters(name = "{index}: {0}")
-  public static Iterable<ParDo.MultiOutput<?, ?>> data() {
-    return ImmutableList.<ParDo.MultiOutput<?, ?>>of(
-        ParDo.of(new DropElementsFn()).withOutputTags(new TupleTag<Void>(), TupleTagList.empty()),
-        ParDo.of(new DropElementsFn())
-            .withOutputTags(new TupleTag<Void>(), TupleTagList.empty())
-            .withSideInputs(singletonSideInput, multimapSideInput),
-        ParDo.of(new DropElementsFn())
-            .withOutputTags(
-                new TupleTag<Void>(),
-                TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {}))
-            .withSideInputs(singletonSideInput, multimapSideInput),
-        ParDo.of(new DropElementsFn())
-            .withOutputTags(
-                new TupleTag<Void>(),
-                TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {})));
-  }
 
-  @Parameter(0)
-  public ParDo.MultiOutput<KV<Long, String>, Void> parDo;
+  /**
+   * Tests for translating various {@link ParDo} transforms to/from {@link ParDoPayload} protos.
+   */
+  @RunWith(Parameterized.class)
+  public static class TestParDoPayloadTranslation {
+    public static TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+    private static PCollectionView<Long> singletonSideInput =
+        p.apply("GenerateSingleton", GenerateSequence.from(0L).to(1L))
+            .apply(View.<Long>asSingleton());
+    private static PCollectionView<Map<Long, Iterable<String>>> multimapSideInput =
+        p.apply("CreateMultimap", Create.of(KV.of(1L, "foo"), KV.of(1L, "bar"), KV.of(2L, "spam")))
+            .setCoder(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of()))
+            .apply(View.<Long, String>asMultimap());
 
-  @Test
-  public void testToAndFromProto() throws Exception {
-    SdkComponents components = SdkComponents.create();
-    ParDoPayload payload = ParDoTranslation.toProto(parDo, components);
+    private static PCollection<KV<Long, String>> mainInput =
+        p.apply(
+            "CreateMainInput", Create.empty(KvCoder.of(VarLongCoder.of(), StringUtf8Coder.of())));
 
-    assertThat(ParDoTranslation.getDoFn(payload), Matchers.<DoFn<?, ?>>equalTo(parDo.getFn()));
-    assertThat(
-        ParDoTranslation.getMainOutputTag(payload),
-        Matchers.<TupleTag<?>>equalTo(parDo.getMainOutputTag()));
-    for (PCollectionView<?> view : parDo.getSideInputs()) {
-      payload.getSideInputsOrThrow(view.getTagInternal().getId());
+    @Parameters(name = "{index}: {0}")
+    public static Iterable<ParDo.MultiOutput<?, ?>> data() {
+      return ImmutableList.<ParDo.MultiOutput<?, ?>>of(
+          ParDo.of(new DropElementsFn()).withOutputTags(new TupleTag<Void>(), TupleTagList.empty()),
+          ParDo.of(new DropElementsFn())
+              .withOutputTags(new TupleTag<Void>(), TupleTagList.empty())
+              .withSideInputs(singletonSideInput, multimapSideInput),
+          ParDo.of(new DropElementsFn())
+              .withOutputTags(
+                  new TupleTag<Void>(),
+                  TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {}))
+              .withSideInputs(singletonSideInput, multimapSideInput),
+          ParDo.of(new DropElementsFn())
+              .withOutputTags(
+                  new TupleTag<Void>(),
+                  TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {})));
     }
-  }
 
-  @Test
-  public void toAndFromTransformProto() throws Exception {
-    Map<TupleTag<?>, PValue> inputs = new HashMap<>();
-    inputs.put(new TupleTag<KV<Long, String>>() {}, mainInput);
-    inputs.putAll(parDo.getAdditionalInputs());
-    PCollectionTuple output = mainInput.apply(parDo);
-
-    SdkComponents components = SdkComponents.create();
-    String transformId =
-        components.registerPTransform(
-            AppliedPTransform.<PCollection<KV<Long, String>>, PCollection<Void>, MultiOutput>of(
-                "foo", inputs, output.expand(), parDo, p),
-            Collections.<AppliedPTransform<?, ?, ?>>emptyList());
-
-    Components protoComponents = components.toComponents();
-    RunnerApi.PTransform protoTransform =
-        protoComponents.getTransformsOrThrow(transformId);
-    ParDoPayload parDoPayload = protoTransform.getSpec().getParameter().unpack(ParDoPayload.class);
-    for (PCollectionView<?> view : parDo.getSideInputs()) {
-      SideInput sideInput = parDoPayload.getSideInputsOrThrow(view.getTagInternal().getId());
-      PCollectionView<?> restoredView =
-          ParDoTranslation.fromProto(
-              sideInput, view.getTagInternal().getId(), protoTransform, protoComponents);
-      assertThat(restoredView.getTagInternal(), equalTo(view.getTagInternal()));
-      assertThat(restoredView.getViewFn(), instanceOf(view.getViewFn().getClass()));
+    @Parameter(0)
+    public ParDo.MultiOutput<KV<Long, String>, Void> parDo;
+
+    @Test
+    public void testToAndFromProto() throws Exception {
+      SdkComponents components = SdkComponents.create();
+      ParDoPayload payload = ParDoTranslation.toProto(parDo, components);
+
+      assertThat(ParDoTranslation.getDoFn(payload), Matchers.<DoFn<?, ?>>equalTo(parDo.getFn()));
       assertThat(
-          restoredView.getWindowMappingFn(), instanceOf(view.getWindowMappingFn().getClass()));
+          ParDoTranslation.getMainOutputTag(payload),
+          Matchers.<TupleTag<?>>equalTo(parDo.getMainOutputTag()));
+      for (PCollectionView<?> view : parDo.getSideInputs()) {
+        payload.getSideInputsOrThrow(view.getTagInternal().getId());
+      }
+    }
+
+    @Test
+    public void toAndFromTransformProto() throws Exception {
+      Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+      inputs.put(new TupleTag<KV<Long, String>>() {}, mainInput);
+      inputs.putAll(parDo.getAdditionalInputs());
+      PCollectionTuple output = mainInput.apply(parDo);
+
+      SdkComponents components = SdkComponents.create();
+      String transformId =
+          components.registerPTransform(
+              AppliedPTransform.<PCollection<KV<Long, String>>, PCollection<Void>, MultiOutput>of(
+                  "foo", inputs, output.expand(), parDo, p),
+              Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+
+      Components protoComponents = components.toComponents();
+      RunnerApi.PTransform protoTransform = protoComponents.getTransformsOrThrow(transformId);
+      ParDoPayload parDoPayload =
+          protoTransform.getSpec().getParameter().unpack(ParDoPayload.class);
+      for (PCollectionView<?> view : parDo.getSideInputs()) {
+        SideInput sideInput = parDoPayload.getSideInputsOrThrow(view.getTagInternal().getId());
+        PCollectionView<?> restoredView =
+            ParDoTranslation.fromProto(
+                sideInput, view.getTagInternal().getId(), protoTransform, protoComponents);
+        assertThat(restoredView.getTagInternal(), equalTo(view.getTagInternal()));
+        assertThat(restoredView.getViewFn(), instanceOf(view.getViewFn().getClass()));
+        assertThat(
+            restoredView.getWindowMappingFn(), instanceOf(view.getWindowMappingFn().getClass()));
+        assertThat(
+            restoredView.getWindowingStrategyInternal(),
+            Matchers.<WindowingStrategy<?, ?>>equalTo(
+                view.getWindowingStrategyInternal().fixDefaults()));
+        assertThat(restoredView.getCoderInternal(), equalTo(view.getCoderInternal()));
+      }
+      String mainInputId = components.registerPCollection(mainInput);
       assertThat(
-          restoredView.getWindowingStrategyInternal(),
-          Matchers.<WindowingStrategy<?, ?>>equalTo(
-              view.getWindowingStrategyInternal().fixDefaults()));
-      assertThat(restoredView.getCoderInternal(), equalTo(view.getCoderInternal()));
+          ParDoTranslation.getMainInput(protoTransform, protoComponents),
+          equalTo(protoComponents.getPcollectionsOrThrow(mainInputId)));
+    }
+  }
+
+  /**
+   * Tests for translating state and timer bits to/from protos.
+   */
+  @RunWith(JUnit4.class)
+  public static class TestStateAndTimerTranslation {
+
+    @Test
+    public void testValueStateSpecToFromProto() throws Exception {
+      SdkComponents sdkComponents = SdkComponents.create();
+      StateSpec<?> stateSpec = StateSpecs.value(VarIntCoder.of());
+      StateSpec<?> deserializedStateSpec =
+          ParDoTranslation.fromProto(
+              ParDoTranslation.toProto(stateSpec, sdkComponents), sdkComponents.toComponents());
+      assertThat(stateSpec, Matchers.<StateSpec<?>>equalTo(deserializedStateSpec));
+    }
+
+    @Test
+    public void testBagStateSpecToFromProto() throws Exception {
+      SdkComponents sdkComponents = SdkComponents.create();
+      StateSpec<?> stateSpec = StateSpecs.bag(VarIntCoder.of());
+      StateSpec<?> deserializedStateSpec =
+          ParDoTranslation.fromProto(
+              ParDoTranslation.toProto(stateSpec, sdkComponents), sdkComponents.toComponents());
+      assertThat(stateSpec, Matchers.<StateSpec<?>>equalTo(deserializedStateSpec));
+    }
+
+    @Test
+    public void testSetStateSpecToFromProto() throws Exception {
+      SdkComponents sdkComponents = SdkComponents.create();
+      StateSpec<?> stateSpec = StateSpecs.set(VarIntCoder.of());
+      StateSpec<?> deserializedStateSpec =
+          ParDoTranslation.fromProto(
+              ParDoTranslation.toProto(stateSpec, sdkComponents), sdkComponents.toComponents());
+      assertThat(stateSpec, Matchers.<StateSpec<?>>equalTo(deserializedStateSpec));
+    }
+
+    @Test
+    public void testMapStateSpecToFromProto() throws Exception {
+      SdkComponents sdkComponents = SdkComponents.create();
+      StateSpec<?> stateSpec = StateSpecs.map(StringUtf8Coder.of(), VarIntCoder.of());
+      StateSpec<?> deserializedStateSpec =
+          ParDoTranslation.fromProto(
+              ParDoTranslation.toProto(stateSpec, sdkComponents), sdkComponents.toComponents());
+      assertThat(stateSpec, Matchers.<StateSpec<?>>equalTo(deserializedStateSpec));
     }
-    String mainInputId = components.registerPCollection(mainInput);
-    assertThat(
-        ParDoTranslation.getMainInput(protoTransform, protoComponents),
-        equalTo(protoComponents.getPcollectionsOrThrow(mainInputId)));
   }
 
   private static class DropElementsFn extends DoFn<KV<Long, String>, Void> {

From 247f9bc1581984d026764b3d433cb594e700bc21 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 26 May 2017 11:00:40 -0700
Subject: [PATCH 357/387] Revert "Roll-forward Include Additional PTransform
 inputs in Transform Nodes"

This reverts commit 2e2ae9cfa581a73864695d15102acadc2750a57a.
---
 .../apex/translation/TranslationContext.java  |   4 +-
 .../core/construction/TransformInputs.java    |  50 ------
 .../construction/TransformInputsTest.java     | 166 ------------------
 .../runners/direct/DirectGraphVisitor.java    |  15 +-
 .../runners/direct/ParDoEvaluatorFactory.java |   9 +-
 ...ttableProcessElementsEvaluatorFactory.java |   2 -
 .../direct/StatefulParDoEvaluatorFactory.java |   1 -
 .../beam/runners/direct/WatermarkManager.java |  17 +-
 .../runners/direct/ParDoEvaluatorTest.java    |   6 +-
 .../flink/FlinkBatchTranslationContext.java   |   3 +-
 .../FlinkStreamingTranslationContext.java     |   3 +-
 .../dataflow/DataflowPipelineTranslator.java  |   5 +-
 .../spark/translation/EvaluationContext.java  |   4 +-
 .../beam/sdk/runners/TransformHierarchy.java  |  28 +--
 14 files changed, 29 insertions(+), 284 deletions(-)
 delete mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
 delete mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index 94d13e177dec..aff3863624c4 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -34,7 +34,6 @@
 import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
 import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
 import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -94,8 +93,7 @@ public Map<TupleTag<?>, PValue> getInputs() {
   }
 
   public <InputT extends PValue> InputT getInput() {
-    return (InputT)
-        Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
+    return (InputT) Iterables.getOnlyElement(getCurrentTransform().getInputs().values());
   }
 
   public Map<TupleTag<?>, PValue> getOutputs() {
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
deleted file mode 100644
index 2baf93a3c128..000000000000
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.collect.ImmutableList;
-import java.util.Collection;
-import java.util.Map;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
-
-/** Utilities for extracting subsets of inputs from an {@link AppliedPTransform}. */
-public class TransformInputs {
-  /**
-   * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link
-   * PTransform#getAdditionalInputs()}.
-   */
-  public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) {
-    ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder();
-    PTransform<?, ?> transform = application.getTransform();
-    for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) {
-      if (!transform.getAdditionalInputs().containsKey(input.getKey())) {
-        mainInputs.add(input.getValue());
-      }
-    }
-    checkArgument(
-        !mainInputs.build().isEmpty() || application.getInputs().isEmpty(),
-        "Expected at least one main input if any inputs exist");
-    return mainInputs.build();
-  }
-}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
deleted file mode 100644
index f5b2c11e7923..000000000000
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.core.construction;
-
-import static org.junit.Assert.assertThat;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.GenerateSequence;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PDone;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.hamcrest.Matchers;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link TransformInputs}. */
-@RunWith(JUnit4.class)
-public class TransformInputsTest {
-  @Rule public TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false);
-  @Rule public ExpectedException thrown = ExpectedException.none();
-
-  @Test
-  public void nonAdditionalInputsWithNoInputSucceeds() {
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "input-free",
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(),
-            pipeline);
-
-    assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>empty());
-  }
-
-  @Test
-  public void nonAdditionalInputsWithOneMainInputSucceeds() {
-    PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L));
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "input-single",
-            Collections.<TupleTag<?>, PValue>singletonMap(new TupleTag<Long>() {}, input),
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(),
-            pipeline);
-
-    assertThat(
-        TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>containsInAnyOrder(input));
-  }
-
-  @Test
-  public void nonAdditionalInputsWithMultipleNonAdditionalInputsSucceeds() {
-    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
-    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
-    allInputs.put(new TupleTag<Integer>() {}, mainInts);
-    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
-    allInputs.put(new TupleTag<Void>() {}, voids);
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "additional-free",
-            allInputs,
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(),
-            pipeline);
-
-    assertThat(
-        TransformInputs.nonAdditionalInputs(transform),
-        Matchers.<PValue>containsInAnyOrder(voids, mainInts));
-  }
-
-  @Test
-  public void nonAdditionalInputsWithAdditionalInputsSucceeds() {
-    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
-    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
-    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
-
-    Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
-    PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
-    allInputs.put(new TupleTag<Integer>() {}, mainInts);
-    PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
-    allInputs.put(
-        new TupleTag<Void>() {}, voids);
-    allInputs.putAll(additionalInputs);
-
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "additional",
-            allInputs,
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(additionalInputs),
-            pipeline);
-
-    assertThat(
-        TransformInputs.nonAdditionalInputs(transform),
-        Matchers.<PValue>containsInAnyOrder(mainInts, voids));
-  }
-
-  @Test
-  public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() {
-    Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
-    additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
-    additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
-
-    AppliedPTransform<PInput, POutput, TestTransform> transform =
-        AppliedPTransform.of(
-            "additional-only",
-            additionalInputs,
-            Collections.<TupleTag<?>, PValue>emptyMap(),
-            new TestTransform(additionalInputs),
-            pipeline);
-
-    thrown.expect(IllegalArgumentException.class);
-    thrown.expectMessage("at least one");
-    TransformInputs.nonAdditionalInputs(transform);
-  }
-
-  private static class TestTransform extends PTransform<PInput, POutput> {
-    private final Map<TupleTag<?>, PValue> additionalInputs;
-
-    private TestTransform() {
-      this(Collections.<TupleTag<?>, PValue>emptyMap());
-    }
-
-    private TestTransform(Map<TupleTag<?>, PValue> additionalInputs) {
-      this.additionalInputs = additionalInputs;
-    }
-
-    @Override
-    public POutput expand(PInput input) {
-      return PDone.in(input.getPipeline());
-    }
-
-    @Override
-    public Map<TupleTag<?>, PValue> getAdditionalInputs() {
-      return additionalInputs;
-    }
-  }
-}
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index ed4282bbdb8b..01204e3049dd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -21,12 +21,10 @@
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -36,8 +34,6 @@
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
 import org.apache.beam.sdk.values.PValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Tracks the {@link AppliedPTransform AppliedPTransforms} that consume each {@link PValue} in the
@@ -45,7 +41,6 @@
  * input after the upstream transform has produced and committed output.
  */
 class DirectGraphVisitor extends PipelineVisitor.Defaults {
-  private static final Logger LOG = LoggerFactory.getLogger(DirectGraphVisitor.class);
 
   private Map<POutput, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
 
@@ -88,15 +83,7 @@ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
     if (node.getInputs().isEmpty()) {
       rootTransforms.add(appliedTransform);
     } else {
-      Collection<PValue> mainInputs =
-          TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline()));
-      if (!mainInputs.containsAll(node.getInputs().values())) {
-        LOG.debug(
-            "Inputs reduced to {} from {} by removing additional inputs",
-            mainInputs,
-            node.getInputs().values());
-      }
-      for (PValue value : mainInputs) {
+      for (PValue value : node.getInputs().values()) {
         primitiveConsumers.put(value, appliedTransform);
       }
     }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 516f798aba97..8aa75cf1445e 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -20,6 +20,7 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -78,7 +79,6 @@ public <T> TransformEvaluator<T> forApplication(
         (TransformEvaluator<T>)
             createEvaluator(
                 (AppliedPTransform) application,
-                (PCollection<InputT>) inputBundle.getPCollection(),
                 inputBundle.getKey(),
                 doFn,
                 transform.getSideInputs(),
@@ -102,7 +102,6 @@ public void cleanup() throws Exception {
   @SuppressWarnings({"unchecked", "rawtypes"})
   DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
-      PCollection<InputT> mainInput,
       StructuralKey<?> inputBundleKey,
       DoFn<InputT, OutputT> doFn,
       List<PCollectionView<?>> sideInputs,
@@ -121,7 +120,6 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
         createParDoEvaluator(
             application,
             inputBundleKey,
-            mainInput,
             sideInputs,
             mainOutputTag,
             additionalOutputTags,
@@ -134,7 +132,6 @@ DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
   ParDoEvaluator<InputT> createParDoEvaluator(
       AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
       StructuralKey<?> key,
-      PCollection<InputT> mainInput,
       List<PCollectionView<?>> sideInputs,
       TupleTag<OutputT> mainOutputTag,
       List<TupleTag<?>> additionalOutputTags,
@@ -147,7 +144,8 @@ ParDoEvaluator<InputT> createParDoEvaluator(
           evaluationContext,
           stepContext,
           application,
-          mainInput.getWindowingStrategy(),
+          ((PCollection<InputT>) Iterables.getOnlyElement(application.getInputs().values()))
+              .getWindowingStrategy(),
           fn,
           key,
           sideInputs,
@@ -175,4 +173,5 @@ static Map<TupleTag<?>, PCollection<?>> pcollections(Map<TupleTag<?>, PValue> ou
     }
     return pcs;
   }
+
 }
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index eccc83a031cb..b85f481c1489 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -116,8 +116,6 @@ public void cleanup() throws Exception {
             delegateFactory.createParDoEvaluator(
                 application,
                 inputBundle.getKey(),
-                (PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>)
-                    inputBundle.getPCollection(),
                 transform.getSideInputs(),
                 transform.getMainOutputTag(),
                 transform.getAdditionalOutputTags().getAll(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 3619d05b47ca..506c84cec639 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -117,7 +117,6 @@ private TransformEvaluator<KeyedWorkItem<K, KV<K, InputT>>> createEvaluator(
     DoFnLifecycleManagerRemovingTransformEvaluator<KV<K, InputT>> delegateEvaluator =
         delegateFactory.createEvaluator(
             (AppliedPTransform) application,
-            (PCollection) inputBundle.getPCollection(),
             inputBundle.getKey(),
             doFn,
             application.getTransform().getUnderlyingParDo().getSideInputs(),
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index b15b52e314de..4f1b8319dc2d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -823,11 +823,10 @@ private Collection<Watermark> getInputProcessingWatermarks(AppliedPTransform<?,
       inputWmsBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
-        Watermark producerOutputWatermark =
-            getTransformWatermark(graph.getProducer(pvalue)).synchronizedProcessingOutputWatermark;
-        inputWmsBuilder.add(producerOutputWatermark);
-      }
+      Watermark producerOutputWatermark =
+          getTransformWatermark(graph.getProducer(pvalue))
+              .synchronizedProcessingOutputWatermark;
+      inputWmsBuilder.add(producerOutputWatermark);
     }
     return inputWmsBuilder.build();
   }
@@ -839,11 +838,9 @@ private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform)
       inputWatermarksBuilder.add(THE_END_OF_TIME);
     }
     for (PValue pvalue : inputs.values()) {
-      if (graph.getPrimitiveConsumers(pvalue).contains(transform)) {
-        Watermark producerOutputWatermark =
-            getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
-        inputWatermarksBuilder.add(producerOutputWatermark);
-      }
+      Watermark producerOutputWatermark =
+          getTransformWatermark(graph.getProducer(pvalue)).outputWatermark;
+      inputWatermarksBuilder.add(producerOutputWatermark);
     }
     List<Watermark> inputCollectionWatermarks = inputWatermarksBuilder.build();
     return inputCollectionWatermarks;
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 8b86bbe50958..09a21ac524a5 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -98,7 +98,7 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
     when(evaluationContext.createBundle(output)).thenReturn(outputBundle);
 
     ParDoEvaluator<Integer> evaluator =
-        createEvaluator(singletonView, fn, inputPc, output);
+        createEvaluator(singletonView, fn, output);
 
     IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
     WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
@@ -132,7 +132,6 @@ public void sideInputsNotReadyResultHasUnprocessedElements() {
   private ParDoEvaluator<Integer> createEvaluator(
       PCollectionView<Integer> singletonView,
       RecorderFn fn,
-      PCollection<Integer> input,
       PCollection<Integer> output) {
     when(
             evaluationContext.createSideInputReader(
@@ -157,7 +156,8 @@ private ParDoEvaluator<Integer> createEvaluator(
         evaluationContext,
         stepContext,
         transform,
-        input.getWindowingStrategy(),
+        ((PCollection<?>) Iterables.getOnlyElement(transform.getInputs().values()))
+            .getWindowingStrategy(),
         fn,
         null /* key */,
         ImmutableList.<PCollectionView<?>>of(singletonView),
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index 6e7019848b19..0439119dfc40 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -20,7 +20,6 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -144,7 +143,7 @@ Map<TupleTag<?>, PValue> getInputs(PTransform<?, ?> transform) {
 
   @SuppressWarnings("unchecked")
   <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
+    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
   }
 
   Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
index 74a5fb971144..ea5f6b3162af 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
@@ -22,7 +22,6 @@
 import com.google.common.collect.Iterables;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -114,7 +113,7 @@ public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collecti
 
   @SuppressWarnings("unchecked")
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
-    return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
+    return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
   }
 
   public <T extends PInput> Map<TupleTag<?>, PValue> getInputs(PTransform<T, ?> transform) {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index fccd018b13d2..af93ef59bba4 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -56,7 +56,6 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.Nullable;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.core.construction.WindowingStrategyTranslation;
 import org.apache.beam.runners.dataflow.BatchViewOverrides.GroupByKeyAndSortValuesOnly;
 import org.apache.beam.runners.dataflow.DataflowRunner.CombineGroupedValues;
@@ -396,9 +395,7 @@ public <InputT extends PInput> Map<TupleTag<?>, PValue> getInputs(
 
     @Override
     public <InputT extends PValue> InputT getInput(PTransform<InputT, ?> transform) {
-      return (InputT)
-          Iterables.getOnlyElement(
-              TransformInputs.nonAdditionalInputs(getCurrentTransform(transform)));
+      return (InputT) Iterables.getOnlyElement(getInputs(transform).values());
     }
 
     @Override
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 0c6c4d1cb660..8102926f6daa 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -26,7 +26,6 @@
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.runners.core.construction.TransformInputs;
 import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.sdk.Pipeline;
@@ -104,8 +103,7 @@ public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
 
   public <T extends PValue> T getInput(PTransform<T, ?> transform) {
     @SuppressWarnings("unchecked")
-    T input =
-        (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
+    T input = (T) Iterables.getOnlyElement(getInputs(transform).values());
     return input;
   }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index bebc306a8d24..630d24cf68f1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import javax.annotation.Nullable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
@@ -67,7 +68,7 @@ public TransformHierarchy() {
     producers = new HashMap<>();
     producerInput = new HashMap<>();
     unexpandedInputs = new HashMap<>();
-    root = new Node();
+    root = new Node(null, null, "", null);
     current = root;
   }
 
@@ -253,37 +254,26 @@ public class Node {
     @VisibleForTesting
     boolean finishedSpecifying = false;
 
-    /**
-     * Creates the root-level node. The root level node has a null enclosing node, a null transform,
-     * an empty map of inputs, and a name equal to the empty string.
-     */
-    private Node() {
-      this.enclosingNode = null;
-      this.transform = null;
-      this.fullName = "";
-      this.inputs = Collections.emptyMap();
-    }
-
     /**
      * Creates a new Node with the given parent and transform.
      *
+     * <p>EnclosingNode and transform may both be null for a root-level node, which holds all other
+     * nodes.
+     *
      * @param enclosingNode the composite node containing this node
      * @param transform the PTransform tracked by this node
      * @param fullName the fully qualified name of the transform
      * @param input the unexpanded input to the transform
      */
     private Node(
-        Node enclosingNode,
-        PTransform<?, ?> transform,
+        @Nullable Node enclosingNode,
+        @Nullable PTransform<?, ?> transform,
         String fullName,
-        PInput input) {
+        @Nullable PInput input) {
       this.enclosingNode = enclosingNode;
       this.transform = transform;
       this.fullName = fullName;
-      ImmutableMap.Builder<TupleTag<?>, PValue> inputs = ImmutableMap.builder();
-      inputs.putAll(input.expand());
-      inputs.putAll(transform.getAdditionalInputs());
-      this.inputs = inputs.build();
+      this.inputs = input == null ? Collections.<TupleTag<?>, PValue>emptyMap() : input.expand();
     }
 
     /**

From 6ad6433ec0c02aec8656e9e3b27f6e0f974f8ece Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Fri, 26 May 2017 11:04:05 -0700
Subject: [PATCH 358/387] Revert "Visit a Transform Hierarchy in Topological
 Order"

This reverts commit bd1dfdf3c8e145a99bcacebd0c64dcf6580f3ffe.
---
 .../beam/runners/spark/SparkRunner.java       | 13 ---
 .../spark/translation/BoundedDataset.java     |  6 --
 .../translation/TransformTranslator.java      |  1 -
 .../spark/translation/StorageLevelTest.java   |  4 +-
 .../beam/sdk/runners/TransformHierarchy.java  | 46 +---------
 .../sdk/runners/TransformHierarchyTest.java   | 86 -------------------
 6 files changed, 6 insertions(+), 150 deletions(-)

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 1be5e13f4f5b..9e2426ef8381 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -21,10 +21,8 @@
 import com.google.common.collect.Iterables;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -51,7 +49,6 @@
 import org.apache.beam.sdk.options.PipelineOptionsValidator;
 import org.apache.beam.sdk.runners.AppliedPTransform;
 import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PCollection;
@@ -339,8 +336,6 @@ public static class Evaluator extends Pipeline.PipelineVisitor.Defaults {
     protected final EvaluationContext ctxt;
     protected final SparkPipelineTranslator translator;
 
-    private final Set<Node> shouldIgnoreChildren = new HashSet<>();
-
     public Evaluator(SparkPipelineTranslator translator, EvaluationContext ctxt) {
       this.translator = translator;
       this.ctxt = ctxt;
@@ -356,7 +351,6 @@ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
           LOG.info("Entering directly-translatable composite transform: '{}'", node.getFullName());
           LOG.debug("Composite transform class: '{}'", transformClass);
           doVisitTransform(node);
-          shouldIgnoreChildren.add(node);
           return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
         }
       }
@@ -398,13 +392,6 @@ protected boolean shouldDefer(TransformHierarchy.Node node) {
 
     @Override
     public void visitPrimitiveTransform(TransformHierarchy.Node node) {
-      Node parent = node.getEnclosingNode();
-      while (!parent.isRootNode()) {
-        if (shouldIgnoreChildren.contains(parent)) {
-          return;
-        }
-        parent = parent.getEnclosingNode();
-      }
       doVisitTransform(node);
     }
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
index a746634008ef..652c75305f0c 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java
@@ -19,7 +19,6 @@
 package org.apache.beam.runners.spark.translation;
 
 import com.google.common.base.Function;
-import com.google.common.base.Joiner;
 import com.google.common.collect.Iterables;
 import java.util.List;
 import javax.annotation.Nullable;
@@ -98,13 +97,8 @@ public WindowedValue<T> apply(byte[] bytes) {
     return windowedValues;
   }
 
-  int timesCached = 0;
   @Override
   public void cache(String storageLevel) {
-    System.out.printf(
-        "Persisting Dataset %s for RDD %s (id %s) at level %s. %s times before%n",
-        this, getRDD(), getRDD().toDebugString(), storageLevel, timesCached++);
-    System.out.println(Joiner.on("\n\t").join(new Throwable().getStackTrace()));
     // populate the rdd if needed
     getRDD().persist(StorageLevel.fromString(storageLevel));
   }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index 6ca12c9ca6cc..742ea83539e2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -407,7 +407,6 @@ public void evaluate(Read.Bounded<T> transform, EvaluationContext context) {
         JavaRDD<WindowedValue<T>> input = new SourceRDD.Bounded<>(
             jsc.sc(), transform.getSource(), runtimeContext, stepName).toJavaRDD();
         // cache to avoid re-evaluation of the source by Spark's lazy DAG evaluation.
-        System.out.println("Evaluating Bounded Read " + transform);
         context.putDataset(transform, new BoundedDataset<>(input.cache()));
       }
 
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
index 8bd6dae98a74..8f2e681c9e91 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
@@ -58,12 +58,12 @@ public static void teardown() {
 
   @Test
   public void test() throws Exception {
-    PCollection<String> pCollection = pipeline.apply("CreateFoo", Create.of("foo"));
+    PCollection<String> pCollection = pipeline.apply(Create.of("foo"));
 
     // by default, the Spark runner doesn't cache the RDD if it accessed only one time.
     // So, to "force" the caching of the RDD, we have to call the RDD at least two time.
     // That's why we are using Count fn on the PCollection.
-    pCollection.apply("CountAll", Count.<String>globally());
+    pCollection.apply(Count.<String>globally());
 
     PCollection<String> output = pCollection.apply(new StorageLevelPTransform());
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 630d24cf68f1..2f0e8efd7de8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -202,12 +202,10 @@ Node getProducer(PValue produced) {
     return producers.get(produced);
   }
 
-  int traversed = 0;
   public Set<PValue> visit(PipelineVisitor visitor) {
     finishSpecifying();
     Set<PValue> visitedValues = new HashSet<>();
-    traversed++;
-    root.visit(visitor, visitedValues, new HashSet<Node>(), new HashSet<Node>());
+    root.visit(visitor, visitedValues);
     return visitedValues;
   }
 
@@ -464,22 +462,7 @@ public Map<TupleTag<?>, PValue> getOutputs() {
      * <p>Provides an ordered visit of the input values, the primitive transform (or child nodes for
      * composite transforms), then the output values.
      */
-    private void visit(
-        PipelineVisitor visitor,
-        Set<PValue> visitedValues,
-        Set<Node> visitedNodes,
-        Set<Node> passedComposites) {
-      if (getEnclosingNode() != null && !visitedNodes.contains(getEnclosingNode())) {
-        getEnclosingNode().visit(visitor, visitedValues, visitedNodes, passedComposites);
-      }
-      if (!visitedNodes.add(this)) {
-        LOG.debug("Not revisiting previously visited node {}", this);
-        return;
-      } else if (childNodeOf(passedComposites)) {
-        LOG.debug("Not revisiting Node {} which is a child of a previously passed composite", this);
-        return;
-      }
-
+    private void visit(PipelineVisitor visitor, Set<PValue> visitedValues) {
       if (!finishedSpecifying) {
         finishSpecifying();
       }
@@ -487,31 +470,22 @@ private void visit(
       if (!isRootNode()) {
         // Visit inputs.
         for (PValue inputValue : inputs.values()) {
-          Node valueProducer = getProducer(inputValue);
-          if (!visitedNodes.contains(valueProducer)) {
-            valueProducer.visit(visitor, visitedValues, visitedNodes, passedComposites);
-          }
           if (visitedValues.add(inputValue)) {
-            LOG.debug("Visiting input value {}", inputValue);
-            visitor.visitValue(inputValue, valueProducer);
+            visitor.visitValue(inputValue, getProducer(inputValue));
           }
         }
       }
 
       if (isCompositeNode()) {
-        LOG.debug("Visiting composite node {}", this);
         PipelineVisitor.CompositeBehavior recurse = visitor.enterCompositeTransform(this);
 
         if (recurse.equals(CompositeBehavior.ENTER_TRANSFORM)) {
           for (Node child : parts) {
-            child.visit(visitor, visitedValues, visitedNodes, passedComposites);
+            child.visit(visitor, visitedValues);
           }
-        } else {
-          passedComposites.add(this);
         }
         visitor.leaveCompositeTransform(this);
       } else {
-        LOG.debug("Visiting primitive node {}", this);
         visitor.visitPrimitiveTransform(this);
       }
 
@@ -520,24 +494,12 @@ private void visit(
         // Visit outputs.
         for (PValue pValue : outputs.values()) {
           if (visitedValues.add(pValue)) {
-            LOG.debug("Visiting output value {}", pValue);
             visitor.visitValue(pValue, this);
           }
         }
       }
     }
 
-    private boolean childNodeOf(Set<Node> nodes) {
-      if (isRootNode()) {
-        return false;
-      }
-      Node parent = this.getEnclosingNode();
-      while (!parent.isRootNode() && !nodes.contains(parent)) {
-        parent = parent.getEnclosingNode();
-      }
-      return nodes.contains(parent);
-    }
-
     /**
      * Finish specifying a transform.
      *
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
index 2fe2817f534c..1197d1b04eb6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
@@ -19,7 +19,6 @@
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertThat;
@@ -33,8 +32,6 @@
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor;
 import org.apache.beam.sdk.Pipeline.PipelineVisitor.Defaults;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.GenerateSequence;
 import org.apache.beam.sdk.io.Read;
@@ -495,87 +492,4 @@ public void visitPrimitiveTransform(Node node) {
     assertThat(visitedPrimitiveNodes, containsInAnyOrder(upstreamNode, replacementParNode));
     assertThat(visitedValues, Matchers.<PValue>containsInAnyOrder(upstream, output));
   }
-
-  @Test
-  public void visitIsTopologicallyOrdered() {
-    PCollection<String> one =
-        PCollection.<String>createPrimitiveOutputInternal(
-                pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)
-            .setCoder(StringUtf8Coder.of());
-    final PCollection<Integer> two =
-        PCollection.<Integer>createPrimitiveOutputInternal(
-                pipeline, WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
-            .setCoder(VarIntCoder.of());
-    final PDone done = PDone.in(pipeline);
-    final TupleTag<String> oneTag = new TupleTag<String>() {};
-    final TupleTag<Integer> twoTag = new TupleTag<Integer>() {};
-    final PCollectionTuple oneAndTwo = PCollectionTuple.of(oneTag, one).and(twoTag, two);
-
-    hierarchy.pushNode("consumes_both", one, new PTransform<PCollection<String>, PDone>() {
-      @Override
-      public PDone expand(PCollection<String> input) {
-        return done;
-      }
-
-      @Override
-      public Map<TupleTag<?>, PValue> getAdditionalInputs() {
-        return Collections.<TupleTag<?>, PValue>singletonMap(twoTag, two);
-      }
-    });
-    hierarchy.setOutput(done);
-    hierarchy.popNode();
-
-    final PTransform<PBegin, PCollectionTuple> producer =
-        new PTransform<PBegin, PCollectionTuple>() {
-          @Override
-          public PCollectionTuple expand(PBegin input) {
-            return oneAndTwo;
-          }
-        };
-    hierarchy.pushNode(
-        "encloses_producer",
-        PBegin.in(pipeline),
-        new PTransform<PBegin, PCollectionTuple>() {
-          @Override
-          public PCollectionTuple expand(PBegin input) {
-            return input.apply(producer);
-          }
-        });
-    hierarchy.pushNode(
-        "creates_one_and_two",
-        PBegin.in(pipeline), producer);
-    hierarchy.setOutput(oneAndTwo);
-    hierarchy.popNode();
-    hierarchy.setOutput(oneAndTwo);
-    hierarchy.popNode();
-
-    hierarchy.visit(new PipelineVisitor.Defaults() {
-      private final Set<Node> visitedNodes = new HashSet<>();
-      private final Set<PValue> visitedValues = new HashSet<>();
-      @Override
-      public CompositeBehavior enterCompositeTransform(Node node) {
-        for (PValue input : node.getInputs().values()) {
-          assertThat(visitedValues, hasItem(input));
-        }
-        visitedNodes.add(node);
-        return CompositeBehavior.ENTER_TRANSFORM;
-      }
-
-      @Override
-      public void visitPrimitiveTransform(Node node) {
-        assertThat(visitedNodes, hasItem(node.getEnclosingNode()));
-        for (PValue input : node.getInputs().values()) {
-          assertThat(visitedValues, hasItem(input));
-        }
-        visitedNodes.add(node);
-      }
-
-      @Override
-      public void visitValue(PValue value, Node producer) {
-        assertThat(visitedNodes, hasItem(producer));
-        assertThat(visitedValues, not(hasItem(value)));
-        visitedValues.add(value);
-      }
-    });
-  }
 }

From a251d01ab26aeada45a7c3824af295071c4d7df4 Mon Sep 17 00:00:00 2001
From: Mairbek Khadikov <mairbek@google.com>
Date: Fri, 26 May 2017 13:46:12 -0700
Subject: [PATCH 359/387] [BEAM-1542] SpannerIO sink updates

* Add unit and integration tests
* Add MutationSizeEstimator
* Improve display data
* Upgrade libraries
---
 pom.xml                                       |  42 +--
 sdks/java/io/google-cloud-platform/pom.xml    |  20 +-
 .../io/gcp/spanner/MutationSizeEstimator.java |  98 +++++++
 .../beam/sdk/io/gcp/spanner/SpannerIO.java    | 182 +++++++++----
 .../spanner/MutationSizeEstimatorTest.java    | 138 ++++++++++
 .../sdk/io/gcp/spanner/SpannerIOTest.java     | 244 ++++++++++++++++++
 .../sdk/io/gcp/spanner/SpannerWriteIT.java    | 159 ++++++++++++
 7 files changed, 808 insertions(+), 75 deletions(-)
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOTest.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java

diff --git a/pom.xml b/pom.xml
index 7e06b5f12952..ba1cf37d0372 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,7 +125,7 @@
     <google-cloud-dataflow-java-proto-library-all.version>0.5.160304</google-cloud-dataflow-java-proto-library-all.version>
     <guava.version>20.0</guava.version>
     <grpc.version>1.2.0</grpc.version>
-    <grpc-google-common-protos.version>0.1.0</grpc-google-common-protos.version>
+    <grpc-google-common-protos.version>0.1.9</grpc-google-common-protos.version>
     <hamcrest.version>1.3</hamcrest.version>
     <jackson.version>2.8.8</jackson.version>
     <findbugs.version>3.0.1</findbugs.version>
@@ -176,8 +176,8 @@
   </modules>
 
   <profiles>
-    <!-- A global profile defined for all modules for release-level verification. 
-      Optional processes such as building source and javadoc should be limited 
+    <!-- A global profile defined for all modules for release-level verification.
+      Optional processes such as building source and javadoc should be limited
       to this profile. -->
     <profile>
       <id>release</id>
@@ -494,7 +494,7 @@
         <artifactId>beam-sdks-java-io-hadoop-input-format</artifactId>
 	    <version>${project.version}</version>
       </dependency>
-	
+
       <dependency>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-runners-core-construction-java</artifactId>
@@ -738,13 +738,13 @@
         <artifactId>google-auth-library-credentials</artifactId>
         <version>${google-auth.version}</version>
       </dependency>
-  
+
       <dependency>
         <groupId>com.google.auth</groupId>
         <artifactId>google-auth-library-oauth2-http</artifactId>
         <version>${google-auth.version}</version>
         <exclusions>
-          <!-- Exclude an old version of guava that is being pulled in by a transitive 
+          <!-- Exclude an old version of guava that is being pulled in by a transitive
             dependency of google-api-client -->
           <exclusion>
             <groupId>com.google.guava</groupId>
@@ -809,12 +809,24 @@
         </exclusions>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.api.grpc</groupId>
+        <artifactId>proto-google-cloud-spanner-admin-database-v1</artifactId>
+        <version>${grpc-google-common-protos.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>com.google.api.grpc</groupId>
+        <artifactId>proto-google-common-protos</artifactId>
+        <version>${grpc-google-common-protos.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.google.apis</groupId>
         <artifactId>google-api-services-storage</artifactId>
         <version>${storage.version}</version>
         <exclusions>
-          <!-- Exclude an old version of guava that is being pulled in by a transitive 
+          <!-- Exclude an old version of guava that is being pulled in by a transitive
             dependency of google-api-client -->
           <exclusion>
             <groupId>com.google.guava</groupId>
@@ -901,7 +913,7 @@
         <artifactId>google-api-services-dataflow</artifactId>
         <version>${dataflow.version}</version>
         <exclusions>
-          <!-- Exclude an old version of guava that is being pulled in by a transitive 
+          <!-- Exclude an old version of guava that is being pulled in by a transitive
             dependency of google-api-client -->
           <exclusion>
             <groupId>com.google.guava</groupId>
@@ -915,7 +927,7 @@
         <artifactId>google-api-services-clouddebugger</artifactId>
         <version>${clouddebugger.version}</version>
         <exclusions>
-          <!-- Exclude an old version of guava that is being pulled in by a transitive 
+          <!-- Exclude an old version of guava that is being pulled in by a transitive
             dependency of google-api-client -->
           <exclusion>
             <groupId>com.google.guava</groupId>
@@ -1016,7 +1028,7 @@
         <artifactId>byte-buddy</artifactId>
         <version>1.6.8</version>
       </dependency>
-      
+
       <dependency>
         <groupId>org.springframework</groupId>
         <artifactId>spring-expression</artifactId>
@@ -1123,7 +1135,7 @@
           <artifactId>maven-antrun-plugin</artifactId>
           <version>1.8</version>
         </plugin>
-        
+
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-checkstyle-plugin</artifactId>
@@ -1394,7 +1406,7 @@
           </configuration>
         </plugin>
 
-        <!-- This plugin's configuration tells the m2e plugin how to import this 
+        <!-- This plugin's configuration tells the m2e plugin how to import this
           Maven project into the Eclipse environment. -->
         <plugin>
           <groupId>org.eclipse.m2e</groupId>
@@ -1731,7 +1743,7 @@
             </goals>
             <configuration>
               <outputDirectory>${basedir}/sdks/python</outputDirectory>
-              <resources>          
+              <resources>
                 <resource>
                   <directory>${basedir}</directory>
                   <includes>
@@ -1740,8 +1752,8 @@
                     <include>README.md</include>
                   </includes>
                 </resource>
-              </resources>              
-            </configuration>            
+              </resources>
+            </configuration>
           </execution>
         </executions>
       </plugin>
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 9143ccf553cf..8b5382092ec6 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -86,11 +86,6 @@
       <artifactId>grpc-core</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.api.grpc</groupId>
-      <artifactId>grpc-google-common-protos</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-bigquery</artifactId>
@@ -248,6 +243,21 @@
       <optional>true</optional>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>proto-google-cloud-spanner-admin-database-v1</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>proto-google-common-protos</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+
     <!--  Test dependencies -->
     <dependency>
       <groupId>org.apache.beam</groupId>
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java
new file mode 100644
index 000000000000..61652e736e90
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import com.google.cloud.ByteArray;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Value;
+
+/** Estimates the logical size of {@link com.google.cloud.spanner.Mutation}. */
+class MutationSizeEstimator {
+
+  // Prevent construction.
+  private MutationSizeEstimator() {}
+
+  /** Estimates a size of mutation in bytes. */
+  static long sizeOf(Mutation m) {
+    long result = 0;
+    for (Value v : m.getValues()) {
+      switch (v.getType().getCode()) {
+        case ARRAY:
+          result += estimateArrayValue(v);
+          break;
+        case STRUCT:
+          throw new IllegalArgumentException("Structs are not supported in mutation.");
+        default:
+          result += estimatePrimitiveValue(v);
+      }
+    }
+    return result;
+  }
+
+  private static long estimatePrimitiveValue(Value v) {
+    switch (v.getType().getCode()) {
+      case BOOL:
+        return 1;
+      case INT64:
+      case FLOAT64:
+        return 8;
+      case DATE:
+      case TIMESTAMP:
+        return 12;
+      case STRING:
+        return v.isNull() ? 0 : v.getString().length();
+      case BYTES:
+        return v.isNull() ? 0 : v.getBytes().length();
+    }
+    throw new IllegalArgumentException("Unsupported type " + v.getType());
+  }
+
+  private static long estimateArrayValue(Value v) {
+    switch (v.getType().getArrayElementType().getCode()) {
+      case BOOL:
+        return v.getBoolArray().size();
+      case INT64:
+        return 8 * v.getInt64Array().size();
+      case FLOAT64:
+        return 8 * v.getFloat64Array().size();
+      case STRING:
+        long totalLength = 0;
+        for (String s : v.getStringArray()) {
+          if (s == null) {
+            continue;
+          }
+          totalLength += s.length();
+        }
+        return totalLength;
+      case BYTES:
+        totalLength = 0;
+        for (ByteArray bytes : v.getBytesArray()) {
+          if (bytes == null) {
+            continue;
+          }
+          totalLength += bytes.length();
+        }
+        return totalLength;
+      case DATE:
+        return 12 * v.getDateArray().size();
+      case TIMESTAMP:
+        return 12 * v.getTimestampArray().size();
+    }
+    throw new IllegalArgumentException("Unsupported type " + v.getType());
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
index ec119311c106..c5325bb5a85f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
@@ -20,6 +20,8 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.auto.value.AutoValue;
+import com.google.cloud.ServiceFactory;
+import com.google.cloud.ServiceOptions;
 import com.google.cloud.spanner.AbortedException;
 import com.google.cloud.spanner.DatabaseClient;
 import com.google.cloud.spanner.DatabaseId;
@@ -32,6 +34,7 @@
 import java.util.List;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -48,8 +51,8 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Experimental {@link PTransform Transforms} for reading from and writing to
- * <a href="https://cloud.google.com/spanner">Google Cloud Spanner</a>.
+ * Experimental {@link PTransform Transforms} for reading from and writing to <a
+ * href="https://cloud.google.com/spanner">Google Cloud Spanner</a>.
  *
  * <h3>Reading from Cloud Spanner</h3>
  *
@@ -72,21 +75,35 @@
  * mutations.apply(
  *     "Write", SpannerIO.write().withInstanceId("instance").withDatabaseId("database"));
  * }</pre>
+ *
+ * <p>The default size of the batch is set to 1MB, to override this use {@link
+ * Write#withBatchSizeBytes(long)}. Setting batch size to a small value or zero practically disables
+ * batching.
+ *
+ * <p>The transform does not provide same transactional guarantees as Cloud Spanner. In particular,
+ *
+ * <ul>
+ *   <li>Mutations are not submitted atomically;
+ *   <li>A mutation is applied at least once;
+ *   <li>If the pipeline was unexpectedly stopped, mutations that were already applied will not get
+ *       rolled back.
+ * </ul>
  */
 @Experimental(Experimental.Kind.SOURCE_SINK)
 public class SpannerIO {
 
-  @VisibleForTesting
-  static final int SPANNER_MUTATIONS_PER_COMMIT_LIMIT = 20000;
+  private static final long DEFAULT_BATCH_SIZE_BYTES = 1024 * 1024; // 1 MB
 
   /**
-   * Creates an unitialized instance of {@link Write}. Before use, the {@link Write} must be
+   * Creates an uninitialized instance of {@link Write}. Before use, the {@link Write} must be
    * configured with a {@link Write#withInstanceId} and {@link Write#withDatabaseId} that identify
    * the Cloud Spanner database being written.
    */
   @Experimental
   public static Write write() {
-    return new AutoValue_SpannerIO_Write.Builder().build();
+    return new AutoValue_SpannerIO_Write.Builder()
+        .setBatchSizeBytes(DEFAULT_BATCH_SIZE_BYTES)
+        .build();
   }
 
   /**
@@ -98,24 +115,57 @@ public static Write write() {
   @AutoValue
   public abstract static class Write extends PTransform<PCollection<Mutation>, PDone> {
 
+    @Nullable
+    abstract String getProjectId();
+
     @Nullable
     abstract String getInstanceId();
 
     @Nullable
     abstract String getDatabaseId();
 
+    abstract long getBatchSizeBytes();
+
+    @Nullable
+    @VisibleForTesting
+    abstract ServiceFactory<Spanner, SpannerOptions> getServiceFactory();
+
     abstract Builder toBuilder();
 
     @AutoValue.Builder
     abstract static class Builder {
 
+      abstract Builder setProjectId(String projectId);
+
       abstract Builder setInstanceId(String instanceId);
 
       abstract Builder setDatabaseId(String databaseId);
 
+      abstract Builder setBatchSizeBytes(long batchSizeBytes);
+
+      @VisibleForTesting
+      abstract Builder setServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory);
+
       abstract Write build();
     }
 
+    SpannerOptions getSpannerOptions() {
+      SpannerOptions.Builder builder = SpannerOptions.newBuilder();
+      if (getServiceFactory() != null) {
+        builder.setServiceFactory(getServiceFactory());
+      }
+      return builder.build();
+    }
+
+    /**
+     * Returns a new {@link SpannerIO.Write} that will write to the specified Cloud Spanner project.
+     *
+     * <p>Does not modify this object.
+     */
+    public Write withProjectId(String projectId) {
+      return toBuilder().setProjectId(projectId).build();
+    }
+
     /**
      * Returns a new {@link SpannerIO.Write} that will write to the specified Cloud Spanner
      * instance.
@@ -126,6 +176,15 @@ public Write withInstanceId(String instanceId) {
       return toBuilder().setInstanceId(instanceId).build();
     }
 
+    /**
+     * Returns a new {@link SpannerIO.Write} with a new batch size limit.
+     *
+     * <p>Does not modify this object.
+     */
+    public Write withBatchSizeBytes(long batchSizeBytes) {
+      return toBuilder().setBatchSizeBytes(batchSizeBytes).build();
+    }
+
     /**
      * Returns a new {@link SpannerIO.Write} that will write to the specified Cloud Spanner
      * database.
@@ -136,12 +195,24 @@ public Write withDatabaseId(String databaseId) {
       return toBuilder().setDatabaseId(databaseId).build();
     }
 
+    @VisibleForTesting
+    Write withServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory) {
+      return toBuilder().setServiceFactory(serviceFactory).build();
+    }
+
     @Override
-    public PDone expand(PCollection<Mutation> input) {
-      input.apply("Write mutations to Spanner",
-          ParDo.of(new SpannerWriterFn(
-              getInstanceId(), getDatabaseId(), SPANNER_MUTATIONS_PER_COMMIT_LIMIT)));
+    public void validate(PipelineOptions options) {
+      checkNotNull(
+          getInstanceId(),
+          "SpannerIO.write() requires instance id to be set with withInstanceId method");
+      checkNotNull(
+          getDatabaseId(),
+          "SpannerIO.write() requires database id to be set with withDatabaseId method");
+    }
 
+    @Override
+    public PDone expand(PCollection<Mutation> input) {
+      input.apply("Write mutations to Cloud Spanner", ParDo.of(new SpannerWriteFn(this)));
       return PDone.in(input.getPipeline());
     }
 
@@ -149,64 +220,69 @@ public PDone expand(PCollection<Mutation> input) {
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
       builder
-          .addIfNotNull(DisplayData.item("instanceId", getInstanceId())
-              .withLabel("Output Instance"))
-          .addIfNotNull(DisplayData.item("databaseId", getDatabaseId())
-              .withLabel("Output Database"));
+          .addIfNotNull(DisplayData.item("projectId", getProjectId()).withLabel("Output Project"))
+          .addIfNotNull(
+              DisplayData.item("instanceId", getInstanceId()).withLabel("Output Instance"))
+          .addIfNotNull(
+              DisplayData.item("databaseId", getDatabaseId()).withLabel("Output Database"))
+          .add(DisplayData.item("batchSizeBytes", getBatchSizeBytes())
+              .withLabel("Batch Size in Bytes"));
+      if (getServiceFactory() != null) {
+        builder.addIfNotNull(
+            DisplayData.item("serviceFactory", getServiceFactory().getClass().getName())
+                .withLabel("Service Factory"));
+      }
     }
   }
 
-  /**
-   * {@link DoFn} that writes {@link Mutation}s to Google Cloud Spanner. Mutations are written in
-   * batches, where the maximum batch size is {@link SpannerIO#SPANNER_MUTATIONS_PER_COMMIT_LIMIT}.
-   *
-   * <p>Commits are non-transactional.  If a commit fails, it will be retried (up to
-   * {@link SpannerWriterFn#MAX_RETRIES} times). This means that the mutation operation should be
-   * idempotent.
-   *
-   * <p>See <a href="https://cloud.google.com/spanner">Google Cloud Spanner documentation</a>.
-   */
+  /** Batches together and writes mutations to Google Cloud Spanner. */
   @VisibleForTesting
-  static class SpannerWriterFn extends DoFn<Mutation, Void> {
-    private static final Logger LOG = LoggerFactory.getLogger(SpannerWriterFn.class);
+  static class SpannerWriteFn extends DoFn<Mutation, Void> {
+    private static final Logger LOG = LoggerFactory.getLogger(SpannerWriteFn.class);
+    private final Write spec;
     private transient Spanner spanner;
-    private final String instanceId;
-    private final String databaseId;
-    private final int batchSize;
     private transient DatabaseClient dbClient;
     // Current batch of mutations to be written.
-    private final List<Mutation> mutations = new ArrayList<>();
+    private List<Mutation> mutations;
+    private long batchSizeBytes = 0;
 
     private static final int MAX_RETRIES = 5;
     private static final FluentBackoff BUNDLE_WRITE_BACKOFF =
         FluentBackoff.DEFAULT
-            .withMaxRetries(MAX_RETRIES).withInitialBackoff(Duration.standardSeconds(5));
+            .withMaxRetries(MAX_RETRIES)
+            .withInitialBackoff(Duration.standardSeconds(5));
 
     @VisibleForTesting
-    SpannerWriterFn(String instanceId, String databaseId, int batchSize) {
-      this.instanceId = checkNotNull(instanceId, "instanceId");
-      this.databaseId = checkNotNull(databaseId, "databaseId");
-      this.batchSize = batchSize;
+    SpannerWriteFn(Write spec) {
+      this.spec = spec;
     }
 
     @Setup
     public void setup() throws Exception {
-        SpannerOptions options = SpannerOptions.newBuilder().build();
-        spanner = options.getService();
-        dbClient = spanner.getDatabaseClient(
-            DatabaseId.of(options.getProjectId(), instanceId, databaseId));
+      spanner = spec.getSpannerOptions().getService();
+      dbClient =
+          spanner.getDatabaseClient(
+              DatabaseId.of(projectId(), spec.getInstanceId(), spec.getDatabaseId()));
+      mutations = new ArrayList<>();
+      batchSizeBytes = 0;
     }
 
     @ProcessElement
     public void processElement(ProcessContext c) throws Exception {
       Mutation m = c.element();
       mutations.add(m);
-      int columnCount = m.asMap().size();
-      if ((mutations.size() + 1) * columnCount >= batchSize) {
+      batchSizeBytes += MutationSizeEstimator.sizeOf(m);
+      if (batchSizeBytes >= spec.getBatchSizeBytes()) {
         flushBatch();
       }
     }
 
+    private String projectId() {
+      return spec.getProjectId() == null
+          ? ServiceOptions.getDefaultProjectId()
+          : spec.getProjectId();
+    }
+
     @FinishBundle
     public void finishBundle() throws Exception {
       if (!mutations.isEmpty()) {
@@ -217,20 +293,20 @@ public void finishBundle() throws Exception {
     @Teardown
     public void teardown() throws Exception {
       if (spanner == null) {
-          return;
+        return;
       }
       spanner.closeAsync().get();
+      spanner = null;
     }
 
     /**
      * Writes a batch of mutations to Cloud Spanner.
      *
-     * <p>If a commit fails, it will be retried up to {@link #MAX_RETRIES} times.
-     * If the retry limit is exceeded, the last exception from Cloud Spanner will be
-     * thrown.
+     * <p>If a commit fails, it will be retried up to {@link #MAX_RETRIES} times. If the retry limit
+     * is exceeded, the last exception from Cloud Spanner will be thrown.
      *
      * @throws AbortedException if the commit fails or IOException or InterruptedException if
-     * backing off between retries fails.
+     *     backing off between retries fails.
      */
     private void flushBatch() throws AbortedException, IOException, InterruptedException {
       LOG.debug("Writing batch of {} mutations", mutations.size());
@@ -247,8 +323,8 @@ private void flushBatch() throws AbortedException, IOException, InterruptedExcep
         } catch (AbortedException exception) {
           // Only log the code and message for potentially-transient errors. The entire exception
           // will be propagated upon the last retry.
-          LOG.error("Error writing to Spanner ({}): {}", exception.getCode(),
-              exception.getMessage());
+          LOG.error(
+              "Error writing to Spanner ({}): {}", exception.getCode(), exception.getMessage());
           if (!BackOffUtils.next(sleeper, backoff)) {
             LOG.error("Aborting after {} retries.", MAX_RETRIES);
             throw exception;
@@ -256,20 +332,16 @@ private void flushBatch() throws AbortedException, IOException, InterruptedExcep
         }
       }
       LOG.debug("Successfully wrote {} mutations", mutations.size());
-      mutations.clear();
+      mutations = new ArrayList<>();
+      batchSizeBytes = 0;
     }
 
     @Override
     public void populateDisplayData(Builder builder) {
       super.populateDisplayData(builder);
-      builder
-          .addIfNotNull(DisplayData.item("instanceId", instanceId)
-              .withLabel("Instance"))
-          .addIfNotNull(DisplayData.item("databaseId", databaseId)
-              .withLabel("Database"));
+      spec.populateDisplayData(builder);
     }
   }
 
   private SpannerIO() {} // Prevent construction.
-
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java
new file mode 100644
index 000000000000..03eb28ed943d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.cloud.ByteArray;
+import com.google.cloud.Date;
+import com.google.cloud.Timestamp;
+import com.google.cloud.spanner.Mutation;
+import java.util.Arrays;
+import org.junit.Test;
+
+/** A set of unit tests for {@link MutationSizeEstimator}. */
+public class MutationSizeEstimatorTest {
+
+  @Test
+  public void primitives() throws Exception {
+    Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build();
+    Mutation float64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(2.9).build();
+    Mutation bool = Mutation.newInsertOrUpdateBuilder("test").set("one").to(false).build();
+
+    assertThat(MutationSizeEstimator.sizeOf(int64), is(8L));
+    assertThat(MutationSizeEstimator.sizeOf(float64), is(8L));
+    assertThat(MutationSizeEstimator.sizeOf(bool), is(1L));
+  }
+
+  @Test
+  public void primitiveArrays() throws Exception {
+    Mutation int64 =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .toInt64Array(new long[] {1L, 2L, 3L})
+            .build();
+    Mutation float64 =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .toFloat64Array(new double[] {1., 2.})
+            .build();
+    Mutation bool =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .toBoolArray(new boolean[] {true, true, false, true})
+            .build();
+
+    assertThat(MutationSizeEstimator.sizeOf(int64), is(24L));
+    assertThat(MutationSizeEstimator.sizeOf(float64), is(16L));
+    assertThat(MutationSizeEstimator.sizeOf(bool), is(4L));
+  }
+
+  @Test
+  public void strings() throws Exception {
+    Mutation emptyString = Mutation.newInsertOrUpdateBuilder("test").set("one").to("").build();
+    Mutation nullString =
+        Mutation.newInsertOrUpdateBuilder("test").set("one").to((String) null).build();
+    Mutation sampleString = Mutation.newInsertOrUpdateBuilder("test").set("one").to("abc").build();
+    Mutation sampleArray =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .toStringArray(Arrays.asList("one", "two", null))
+            .build();
+
+    assertThat(MutationSizeEstimator.sizeOf(emptyString), is(0L));
+    assertThat(MutationSizeEstimator.sizeOf(nullString), is(0L));
+    assertThat(MutationSizeEstimator.sizeOf(sampleString), is(3L));
+    assertThat(MutationSizeEstimator.sizeOf(sampleArray), is(6L));
+  }
+
+  @Test
+  public void bytes() throws Exception {
+    Mutation empty =
+        Mutation.newInsertOrUpdateBuilder("test").set("one").to(ByteArray.fromBase64("")).build();
+    Mutation nullValue =
+        Mutation.newInsertOrUpdateBuilder("test").set("one").to((ByteArray) null).build();
+    Mutation sample =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .to(ByteArray.fromBase64("abcdabcd"))
+            .build();
+
+    assertThat(MutationSizeEstimator.sizeOf(empty), is(0L));
+    assertThat(MutationSizeEstimator.sizeOf(nullValue), is(0L));
+    assertThat(MutationSizeEstimator.sizeOf(sample), is(6L));
+  }
+
+  @Test
+  public void dates() throws Exception {
+    Mutation timestamp =
+        Mutation.newInsertOrUpdateBuilder("test").set("one").to(Timestamp.now()).build();
+    Mutation nullTimestamp =
+        Mutation.newInsertOrUpdateBuilder("test").set("one").to((Timestamp) null).build();
+    Mutation date =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .to(Date.fromYearMonthDay(2017, 10, 10))
+            .build();
+    Mutation nullDate =
+        Mutation.newInsertOrUpdateBuilder("test").set("one").to((Date) null).build();
+    Mutation timestampArray =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .toTimestampArray(Arrays.asList(Timestamp.now(), null))
+            .build();
+    Mutation dateArray =
+        Mutation.newInsertOrUpdateBuilder("test")
+            .set("one")
+            .toDateArray(
+                Arrays.asList(
+                    null,
+                    Date.fromYearMonthDay(2017, 1, 1),
+                    null,
+                    Date.fromYearMonthDay(2017, 1, 2)))
+            .build();
+
+    assertThat(MutationSizeEstimator.sizeOf(timestamp), is(12L));
+    assertThat(MutationSizeEstimator.sizeOf(date), is(12L));
+    assertThat(MutationSizeEstimator.sizeOf(nullTimestamp), is(12L));
+    assertThat(MutationSizeEstimator.sizeOf(nullDate), is(12L));
+    assertThat(MutationSizeEstimator.sizeOf(timestampArray), is(24L));
+    assertThat(MutationSizeEstimator.sizeOf(dateArray), is(48L));
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOTest.java
new file mode 100644
index 000000000000..5bdfea5522b2
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+
+import com.google.api.core.ApiFuture;
+import com.google.cloud.ServiceFactory;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.common.collect.Iterables;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import javax.annotation.concurrent.GuardedBy;
+
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFnTester;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Matchers;
+
+
+/**
+ * Unit tests for {@link SpannerIO}.
+ */
+@RunWith(JUnit4.class)
+public class SpannerIOTest implements Serializable {
+  @Rule public final transient TestPipeline pipeline = TestPipeline.create();
+  @Rule public transient ExpectedException thrown = ExpectedException.none();
+
+  private FakeServiceFactory serviceFactory;
+
+  @Before
+  @SuppressWarnings("unchecked")
+  public void setUp() throws Exception {
+    serviceFactory = new FakeServiceFactory();
+  }
+
+  @Test
+  public void emptyTransform() throws Exception {
+    SpannerIO.Write write = SpannerIO.write();
+    thrown.expect(NullPointerException.class);
+    thrown.expectMessage("requires instance id to be set with");
+    write.validate(null);
+  }
+
+  @Test
+  public void emptyInstanceId() throws Exception {
+    SpannerIO.Write write = SpannerIO.write().withDatabaseId("123");
+    thrown.expect(NullPointerException.class);
+    thrown.expectMessage("requires instance id to be set with");
+    write.validate(null);
+  }
+
+  @Test
+  public void emptyDatabaseId() throws Exception {
+    SpannerIO.Write write = SpannerIO.write().withInstanceId("123");
+    thrown.expect(NullPointerException.class);
+    thrown.expectMessage("requires database id to be set with");
+    write.validate(null);
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void singleMutationPipeline() throws Exception {
+    Mutation mutation = Mutation.newInsertOrUpdateBuilder("test").set("one").to(2).build();
+    PCollection<Mutation> mutations = pipeline.apply(Create.of(mutation));
+
+    mutations.apply(
+        SpannerIO.write()
+            .withProjectId("test-project")
+            .withInstanceId("test-instance")
+            .withDatabaseId("test-database")
+            .withServiceFactory(serviceFactory));
+    pipeline.run();
+    verify(serviceFactory.mockSpanner())
+        .getDatabaseClient(DatabaseId.of("test-project", "test-instance", "test-database"));
+    verify(serviceFactory.mockDatabaseClient(), times(1))
+        .writeAtLeastOnce(argThat(new IterableOfSize(1)));
+  }
+
+  @Test
+  public void batching() throws Exception {
+    Mutation one = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build();
+    Mutation two = Mutation.newInsertOrUpdateBuilder("test").set("two").to(2).build();
+    SpannerIO.Write write =
+        SpannerIO.write()
+            .withProjectId("test-project")
+            .withInstanceId("test-instance")
+            .withDatabaseId("test-database")
+            .withBatchSizeBytes(1000000000)
+            .withServiceFactory(serviceFactory);
+    SpannerIO.SpannerWriteFn writerFn = new SpannerIO.SpannerWriteFn(write);
+    DoFnTester<Mutation, Void> fnTester = DoFnTester.of(writerFn);
+    fnTester.processBundle(Arrays.asList(one, two));
+
+    verify(serviceFactory.mockSpanner())
+        .getDatabaseClient(DatabaseId.of("test-project", "test-instance", "test-database"));
+    verify(serviceFactory.mockDatabaseClient(), times(1))
+        .writeAtLeastOnce(argThat(new IterableOfSize(2)));
+  }
+
+  @Test
+  public void batchingGroups() throws Exception {
+    Mutation one = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build();
+    Mutation two = Mutation.newInsertOrUpdateBuilder("test").set("two").to(2).build();
+    Mutation three = Mutation.newInsertOrUpdateBuilder("test").set("three").to(3).build();
+
+    // Have a room to accumulate one more item.
+    long batchSize = MutationSizeEstimator.sizeOf(one) + 1;
+
+    SpannerIO.Write write =
+        SpannerIO.write()
+            .withProjectId("test-project")
+            .withInstanceId("test-instance")
+            .withDatabaseId("test-database")
+            .withBatchSizeBytes(batchSize)
+            .withServiceFactory(serviceFactory);
+    SpannerIO.SpannerWriteFn writerFn = new SpannerIO.SpannerWriteFn(write);
+    DoFnTester<Mutation, Void> fnTester = DoFnTester.of(writerFn);
+    fnTester.processBundle(Arrays.asList(one, two, three));
+
+    verify(serviceFactory.mockSpanner())
+        .getDatabaseClient(DatabaseId.of("test-project", "test-instance", "test-database"));
+    verify(serviceFactory.mockDatabaseClient(), times(1))
+        .writeAtLeastOnce(argThat(new IterableOfSize(2)));
+    verify(serviceFactory.mockDatabaseClient(), times(1))
+        .writeAtLeastOnce(argThat(new IterableOfSize(1)));
+  }
+
+  @Test
+  public void noBatching() throws Exception {
+    Mutation one = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build();
+    Mutation two = Mutation.newInsertOrUpdateBuilder("test").set("two").to(2).build();
+    SpannerIO.Write write =
+        SpannerIO.write()
+            .withProjectId("test-project")
+            .withInstanceId("test-instance")
+            .withDatabaseId("test-database")
+            .withBatchSizeBytes(0) // turn off batching.
+            .withServiceFactory(serviceFactory);
+    SpannerIO.SpannerWriteFn writerFn = new SpannerIO.SpannerWriteFn(write);
+    DoFnTester<Mutation, Void> fnTester = DoFnTester.of(writerFn);
+    fnTester.processBundle(Arrays.asList(one, two));
+
+    verify(serviceFactory.mockSpanner())
+        .getDatabaseClient(DatabaseId.of("test-project", "test-instance", "test-database"));
+    verify(serviceFactory.mockDatabaseClient(), times(2))
+        .writeAtLeastOnce(argThat(new IterableOfSize(1)));
+  }
+
+  private static class FakeServiceFactory
+      implements ServiceFactory<Spanner, SpannerOptions>, Serializable {
+    // Marked as static so they could be returned by serviceFactory, which is serializable.
+    private static final Object lock = new Object();
+
+    @GuardedBy("lock")
+    private static final List<Spanner> mockSpanners = new ArrayList<>();
+
+    @GuardedBy("lock")
+    private static final List<DatabaseClient> mockDatabaseClients = new ArrayList<>();
+
+    @GuardedBy("lock")
+    private static int count = 0;
+
+    private final int index;
+
+    public FakeServiceFactory() {
+      synchronized (lock) {
+        index = count++;
+        mockSpanners.add(mock(Spanner.class, withSettings().serializable()));
+        mockDatabaseClients.add(mock(DatabaseClient.class, withSettings().serializable()));
+      }
+      ApiFuture voidFuture = mock(ApiFuture.class, withSettings().serializable());
+      when(mockSpanner().getDatabaseClient(Matchers.any(DatabaseId.class)))
+          .thenReturn(mockDatabaseClient());
+      when(mockSpanner().closeAsync()).thenReturn(voidFuture);
+    }
+
+    DatabaseClient mockDatabaseClient() {
+      synchronized (lock) {
+        return mockDatabaseClients.get(index);
+      }
+    }
+
+    Spanner mockSpanner() {
+      synchronized (lock) {
+        return mockSpanners.get(index);
+      }
+    }
+
+    @Override
+    public Spanner create(SpannerOptions serviceOptions) {
+      return mockSpanner();
+    }
+  }
+
+  private static class IterableOfSize extends ArgumentMatcher<Iterable<Mutation>> {
+    private final int size;
+
+    private IterableOfSize(int size) {
+      this.size = size;
+    }
+
+    @Override
+    public boolean matches(Object argument) {
+      return argument instanceof Iterable && Iterables.size((Iterable<?>) argument) == size;
+    }
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java
new file mode 100644
index 000000000000..064c65eedcef
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.spanner;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Mutation;
+import com.google.cloud.spanner.Operation;
+import com.google.cloud.spanner.ResultSet;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.cloud.spanner.Statement;
+import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
+import java.util.Collections;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestPipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** End-to-end test of Cloud Spanner Sink. */
+@RunWith(JUnit4.class)
+public class SpannerWriteIT {
+  @Rule public final transient TestPipeline p = TestPipeline.create();
+
+  /** Pipeline options for this test. */
+  public interface SpannerTestPipelineOptions extends TestPipelineOptions {
+    @Description("Project ID for Spanner")
+    @Default.String("apache-beam-testing")
+    String getProjectId();
+    void setProjectId(String value);
+
+    @Description("Instance ID to write to in Spanner")
+    @Default.String("beam-test")
+    String getInstanceId();
+    void setInstanceId(String value);
+
+    @Description("Database ID to write to in Spanner")
+    @Default.String("beam-testdb")
+    String getDatabaseId();
+    void setDatabaseId(String value);
+
+    @Description("Table name")
+    @Default.String("users")
+    String getTable();
+    void setTable(String value);
+  }
+
+  private Spanner spanner;
+  private DatabaseAdminClient databaseAdminClient;
+  private SpannerTestPipelineOptions options;
+
+  @Before
+  public void setUp() throws Exception {
+    PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
+    options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);
+
+    spanner = SpannerOptions.newBuilder().setProjectId(options.getProjectId()).build().getService();
+
+    databaseAdminClient = spanner.getDatabaseAdminClient();
+
+    // Delete database if exists.
+    databaseAdminClient.dropDatabase(options.getInstanceId(), options.getDatabaseId());
+
+    Operation<Database, CreateDatabaseMetadata> op =
+        databaseAdminClient.createDatabase(
+            options.getInstanceId(),
+            options.getDatabaseId(),
+            Collections.singleton(
+                "CREATE TABLE "
+                    + options.getTable()
+                    + " ("
+                    + "  Key           INT64,"
+                    + "  Value         STRING(MAX),"
+                    + ") PRIMARY KEY (Key)"));
+    op.waitFor();
+  }
+
+  @Test
+  public void testWrite() throws Exception {
+    p.apply(GenerateSequence.from(0).to(100))
+        .apply(ParDo.of(new GenerateMutations(options.getTable())))
+        .apply(
+            SpannerIO.write()
+                .withProjectId(options.getProjectId())
+                .withInstanceId(options.getInstanceId())
+                .withDatabaseId(options.getDatabaseId()));
+
+    p.run();
+    DatabaseClient databaseClient =
+        spanner.getDatabaseClient(
+            DatabaseId.of(
+                options.getProjectId(), options.getInstanceId(), options.getDatabaseId()));
+
+    ResultSet resultSet =
+        databaseClient
+            .singleUse()
+            .executeQuery(Statement.of("SELECT COUNT(*) FROM " + options.getTable()));
+    assertThat(resultSet.next(), is(true));
+    assertThat(resultSet.getLong(0), equalTo(100L));
+    assertThat(resultSet.next(), is(false));
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    databaseAdminClient.dropDatabase(options.getInstanceId(), options.getDatabaseId());
+    spanner.closeAsync().get();
+  }
+
+  private static class GenerateMutations extends DoFn<Long, Mutation> {
+    private final String table;
+    private final int valueSize = 100;
+
+    public GenerateMutations(String table) {
+      this.table = table;
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext c) {
+      Mutation.WriteBuilder builder = Mutation.newInsertOrUpdateBuilder(table);
+      Long key = c.element();
+      builder.set("Key").to(key);
+      builder.set("Value").to(RandomStringUtils.random(valueSize, true, true));
+      Mutation mutation = builder.build();
+      c.output(mutation);
+    }
+  }
+}

From 6ec64b90b66de8693c833145c583ecb9cb2ae0f5 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 25 May 2017 16:17:54 -0700
Subject: [PATCH 360/387] [BEAM-2366] Don't try to pickle gen_protos in tests.

Tests that pickle the main session but are run from setup.py attempt
to pickle everything visible in setup.py.
---
 sdks/python/setup.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 8a8ce482742a..596c8c51afb0 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -36,8 +36,6 @@
 
 from pkg_resources import get_distribution, DistributionNotFound
 
-import gen_protos
-
 
 def get_version():
   global_names = {}
@@ -127,6 +125,9 @@ def get_version():
 
 # We must generate protos after setup_requires are installed.
 def generate_protos_first(original_cmd):
+  # See https://issues.apache.org/jira/browse/BEAM-2366
+  # pylint: disable=wrong-import-position
+  import gen_protos
   class cmd(original_cmd, object):
     def run(self):
       gen_protos.generate_proto_files()

From ba142aa82713a24f3d0d5b93cc13b957d1192a99 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 26 May 2017 15:13:20 -0700
Subject: [PATCH 361/387] Add Missing Space in DataflowMetrics

---
 .../java/org/apache/beam/runners/dataflow/DataflowMetrics.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
index a6a6a43e3f65..f038e3f8fc6e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
@@ -149,7 +149,7 @@ private MetricQueryResults populateMetricQueryResults(
             || metricName.endsWith("[MEAN]") || metricName.endsWith("[COUNT]")) {
           // Skip distribution metrics, as these are not yet properly supported.
           LOG.warn("Distribution metrics are not yet supported. You can see them in the Dataflow"
-              + "User Interface");
+              + " User Interface");
           continue;
         }
 

From 3878556cdf575e11fb47d0b9c1550cdfc2b60131 Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Fri, 26 May 2017 11:40:38 -0700
Subject: [PATCH 362/387] [BEAM-2372] Only run Apache RAT at root pom.xml

Since it checks the entire project anyway.
---
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pom.xml b/pom.xml
index ba1cf37d0372..398cda09d9d3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -243,6 +243,8 @@
           <plugin>
             <groupId>org.apache.rat</groupId>
             <artifactId>apache-rat-plugin</artifactId>
+            <!-- Apache RAT checks all files in the project, only run once. -->
+            <inherited>false</inherited>
           </plugin>
 
           <plugin>

From 4d86cece9b27c7db13bb7c3ae3f0249e16273ac2 Mon Sep 17 00:00:00 2001
From: Michael Luckey <michael.luckey@ext.gfk.com>
Date: Fri, 26 May 2017 15:52:42 +0200
Subject: [PATCH 363/387] [BEAM-2369] HadoopFileSystem: prevent NPE on match of
 non existing file

---
 .../beam/sdk/io/hdfs/HadoopFileSystem.java    |  7 ++++
 .../sdk/io/hdfs/HadoopFileSystemTest.java     | 33 +++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
index d519a8c86266..803ddb6c1ec3 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
@@ -28,6 +28,7 @@
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import org.apache.beam.sdk.io.FileSystem;
 import org.apache.beam.sdk.io.fs.CreateOptions;
@@ -78,6 +79,12 @@ protected List<MatchResult> match(List<String> specs) {
     for (String spec : specs) {
       try {
         FileStatus[] fileStatuses = fileSystem.globStatus(new Path(spec));
+        if (fileStatuses == null) {
+          resultsBuilder.add(MatchResult.create(Status.NOT_FOUND,
+                  Collections.<Metadata>emptyList()));
+          continue;
+        }
+
         List<Metadata> metadata = new ArrayList<>();
         for (FileStatus fileStatus : fileStatuses) {
           if (fileStatus.isFile()) {
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
index 88275f49b05c..d06142dce84f 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
@@ -19,6 +19,8 @@
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
@@ -157,6 +159,37 @@ public void testMatch() throws Exception {
             .build()));
   }
 
+  @Test
+  public void testMatchForNonExistentFile() throws Exception {
+    create("testFileAA", "testDataAA".getBytes());
+    create("testFileBB", "testDataBB".getBytes());
+
+    // ensure files exist
+    assertArrayEquals("testDataAA".getBytes(), read("testFileAA"));
+    assertArrayEquals("testDataBB".getBytes(), read("testFileBB"));
+
+    List<MatchResult> matchResults = fileSystem.match(ImmutableList.of(
+        testPath("testFileAA").toString(),
+        testPath("testFileA").toString(),
+        testPath("testFileBB").toString()));
+
+    assertThat(matchResults, hasSize(3));
+
+    final List<MatchResult> expected = ImmutableList.of(
+        MatchResult.create(Status.OK, ImmutableList.of(Metadata.builder()
+            .setResourceId(testPath("testFileAA"))
+            .setIsReadSeekEfficient(true)
+            .setSizeBytes("testDataAA".getBytes().length)
+            .build())),
+        MatchResult.create(Status.NOT_FOUND, ImmutableList.<Metadata>of()),
+        MatchResult.create(Status.OK, ImmutableList.of(Metadata.builder()
+            .setResourceId(testPath("testFileBB"))
+            .setIsReadSeekEfficient(true)
+            .setSizeBytes("testDataBB".getBytes().length)
+            .build())));
+    assertThat(matchResults, equalTo(expected));
+  }
+
   @Test
   public void testRename() throws Exception {
     create("testFileA", "testDataA".getBytes());

From 86912a23795353111934365ed9065a2a71f70074 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@gmail.com>
Date: Thu, 25 May 2017 13:49:48 -0700
Subject: [PATCH 364/387] [BEAM-2365] Use the highest pickle protocol
 available.

---
 sdks/python/apache_beam/coders/coders.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index ce914ddec0e0..f40045d142ff 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -365,7 +365,7 @@ def maybe_dill_dumps(o):
   # We need to use the dill pickler for objects of certain custom classes,
   # including, for example, ones that contain lambdas.
   try:
-    return pickle.dumps(o)
+    return pickle.dumps(o, pickle.HIGHEST_PROTOCOL)
   except Exception:  # pylint: disable=broad-except
     return dill.dumps(o)
 
@@ -426,7 +426,10 @@ class PickleCoder(_PickleCoderBase):
   """Coder using Python's pickle functionality."""
 
   def _create_impl(self):
-    return coder_impl.CallbackCoderImpl(pickle.dumps, pickle.loads)
+    dumps = pickle.dumps
+    HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL
+    return coder_impl.CallbackCoderImpl(
+        lambda x: dumps(x, HIGHEST_PROTOCOL), pickle.loads)
 
 
 class DillCoder(_PickleCoderBase):
@@ -515,7 +518,7 @@ class Base64PickleCoder(Coder):
   # than via a special Coder.
 
   def encode(self, value):
-    return base64.b64encode(pickle.dumps(value))
+    return base64.b64encode(pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
 
   def decode(self, encoded):
     return pickle.loads(base64.b64decode(encoded))

From f6ed520fbcb1f2d4b13800b597ecd26eff9c5a08 Mon Sep 17 00:00:00 2001
From: Charles Chen <ccy@google.com>
Date: Fri, 26 May 2017 14:24:55 -0700
Subject: [PATCH 365/387] Fix Python Dataflow execution errors due to #3223

---
 sdks/python/apache_beam/io/gcp/pubsub.py | 74 +++++++++++++++++++++++-
 1 file changed, 73 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index 7b838d23fea6..1ba8ac051272 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -18,10 +18,13 @@
 
 Cloud Pub/Sub sources and sinks are currently supported only in streaming
 pipelines, during remote execution.
+
+This API is currently under development and is subject to change.
 """
 
 from __future__ import absolute_import
 
+from apache_beam import coders
 from apache_beam.io.iobase import Read
 from apache_beam.io.iobase import Write
 from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
@@ -30,7 +33,8 @@
 from apache_beam.transforms.display import DisplayDataItem
 
 
-__all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub']
+__all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub',
+           'PubSubSource', 'PubSubSink']
 
 
 class ReadStringsFromPubSub(PTransform):
@@ -150,3 +154,71 @@ def _decodeUtf8String(encoded_value):
 def _encodeUtf8String(value):
   """Encodes a string in utf-8 format to bytes"""
   return value.encode('utf-8')
+
+
+class PubSubSource(dataflow_io.NativeSource):
+  """Deprecated: do not use.
+
+  Source for reading from a given Cloud Pub/Sub topic.
+
+  Attributes:
+    topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
+    subscription: Optional existing Cloud Pub/Sub subscription to use in the
+      form "projects/<project>/subscriptions/<subscription>".
+    id_label: The attribute on incoming Pub/Sub messages to use as a unique
+      record identifier.  When specified, the value of this attribute (which can
+      be any string that uniquely identifies the record) will be used for
+      deduplication of messages.  If not provided, Dataflow cannot guarantee
+      that no duplicate data will be delivered on the Pub/Sub stream. In this
+      case, deduplication of the stream will be strictly best effort.
+    coder: The Coder to use for decoding incoming Pub/Sub messages.
+  """
+
+  def __init__(self, topic, subscription=None, id_label=None,
+               coder=coders.StrUtf8Coder()):
+    self.topic = topic
+    self.subscription = subscription
+    self.id_label = id_label
+    self.coder = coder
+
+  @property
+  def format(self):
+    """Source format name required for remote execution."""
+    return 'pubsub'
+
+  def display_data(self):
+    return {'id_label':
+            DisplayDataItem(self.id_label,
+                            label='ID Label Attribute').drop_if_none(),
+            'topic':
+            DisplayDataItem(self.topic,
+                            label='Pubsub Topic'),
+            'subscription':
+            DisplayDataItem(self.subscription,
+                            label='Pubsub Subscription').drop_if_none()}
+
+  def reader(self):
+    raise NotImplementedError(
+        'PubSubSource is not supported in local execution.')
+
+
+class PubSubSink(dataflow_io.NativeSink):
+  """Deprecated: do not use.
+
+  Sink for writing to a given Cloud Pub/Sub topic."""
+
+  def __init__(self, topic, coder=coders.StrUtf8Coder()):
+    self.topic = topic
+    self.coder = coder
+
+  @property
+  def format(self):
+    """Sink format name required for remote execution."""
+    return 'pubsub'
+
+  def display_data(self):
+    return {'topic': DisplayDataItem(self.topic, label='Pubsub Topic')}
+
+  def writer(self):
+    raise NotImplementedError(
+        'PubSubSink is not supported in local execution.')

From ac8d7321c1abc4d4ef5cb8f6f699d1ebb57468a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jean-Baptiste=20Onofr=C3=A9?= <jbonofre@apache.org>
Date: Tue, 9 Aug 2016 11:17:32 +0200
Subject: [PATCH 366/387] Add DSLs module

---
 dsls/pom.xml | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 pom.xml      |  1 +
 2 files changed, 57 insertions(+)
 create mode 100644 dsls/pom.xml

diff --git a/dsls/pom.xml b/dsls/pom.xml
new file mode 100644
index 000000000000..6e0017115a41
--- /dev/null
+++ b/dsls/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.beam</groupId>
+    <artifactId>beam-parent</artifactId>
+    <version>0.7.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>beam-dsls-parent</artifactId>
+  <name>Apache Beam :: DSLs</name>
+
+  <modules>
+    <!-- <module>sql</module> -->
+  </modules>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <!-- DSLs will generally offer test suites for runners, as sdks/java does. -->
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-jar-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>default-test-jar</id>
+              <goals>
+                <goal>test-jar</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+</project>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 398cda09d9d3..53017bda0bcd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -170,6 +170,7 @@
     <module>sdks/java/build-tools</module>
     <module>sdks</module>
     <module>runners</module>
+    <module>dsls</module>
     <module>examples</module>
     <!-- sdks/java/javadoc builds project-wide Javadoc. It has to run last. -->
     <module>sdks/java/javadoc</module>

From 780da13f19c404523fc8d5c6996f479a98d26e81 Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Sun, 9 Apr 2017 19:49:08 -0700
Subject: [PATCH 367/387] [BEAM-301] Initial skeleton for Beam SQL

---
 dsls/pom.xml                                  |   5 +-
 dsls/sql/README.md                            |  24 ++
 dsls/sql/pom.xml                              | 166 ++++++++++++
 .../beam/dsls/sql/example/BeamSqlExample.java | 102 ++++++++
 .../beam/dsls/sql/example/package-info.java   |  23 ++
 .../BeamSQLExpressionExecutor.java            |  43 ++++
 .../sql/interpreter/BeamSQLSpELExecutor.java  | 126 +++++++++
 .../dsls/sql/interpreter/CalciteToSpEL.java   |  80 ++++++
 .../dsls/sql/interpreter/package-info.java    |  22 ++
 .../java/org/beam/dsls/sql/package-info.java  |  22 ++
 .../dsls/sql/planner/BeamPipelineCreator.java |  85 ++++++
 .../dsls/sql/planner/BeamQueryPlanner.java    | 157 ++++++++++++
 .../sql/planner/BeamRelDataTypeSystem.java    |  40 +++
 .../beam/dsls/sql/planner/BeamRuleSets.java   |  65 +++++
 .../dsls/sql/planner/BeamSQLRelUtils.java     |  73 ++++++
 .../beam/dsls/sql/planner/BeamSqlRunner.java  |  93 +++++++
 .../planner/BeamSqlUnsupportedException.java  |  38 +++
 .../planner/UnsupportedOperatorsVisitor.java  |  28 ++
 .../beam/dsls/sql/planner/package-info.java   |  24 ++
 .../org/beam/dsls/sql/rel/BeamFilterRel.java  |  71 +++++
 .../org/beam/dsls/sql/rel/BeamIOSinkRel.java  |  75 ++++++
 .../beam/dsls/sql/rel/BeamIOSourceRel.java    |  59 +++++
 .../dsls/sql/rel/BeamLogicalConvention.java   |  72 ++++++
 .../org/beam/dsls/sql/rel/BeamProjectRel.java |  82 ++++++
 .../org/beam/dsls/sql/rel/BeamRelNode.java    |  38 +++
 .../org/beam/dsls/sql/rel/package-info.java   |  23 ++
 .../beam/dsls/sql/rule/BeamFilterRule.java    |  49 ++++
 .../beam/dsls/sql/rule/BeamIOSinkRule.java    |  81 ++++++
 .../beam/dsls/sql/rule/BeamIOSourceRule.java  |  49 ++++
 .../beam/dsls/sql/rule/BeamProjectRule.java   |  50 ++++
 .../org/beam/dsls/sql/rule/package-info.java  |  22 ++
 .../beam/dsls/sql/schema/BaseBeamTable.java   |  99 +++++++
 .../org/beam/dsls/sql/schema/BeamIOType.java  |  28 ++
 .../dsls/sql/schema/BeamSQLRecordType.java    |  74 ++++++
 .../sql/schema/BeamSQLRecordTypeCoder.java    |  88 +++++++
 .../org/beam/dsls/sql/schema/BeamSQLRow.java  | 242 ++++++++++++++++++
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java | 149 +++++++++++
 .../sql/schema/InvalidFieldException.java     |  30 +++
 .../schema/UnsupportedDataTypeException.java  |  28 ++
 .../sql/schema/kafka/BeamKafkaCSVTable.java   | 127 +++++++++
 .../dsls/sql/schema/kafka/BeamKafkaTable.java | 111 ++++++++
 .../dsls/sql/schema/kafka/package-info.java   |  22 ++
 .../beam/dsls/sql/schema/package-info.java    |  23 ++
 .../dsls/sql/transform/BeamSQLFilterFn.java   |  66 +++++
 .../transform/BeamSQLOutputToConsoleFn.java   |  45 ++++
 .../dsls/sql/transform/BeamSQLProjectFn.java  |  72 ++++++
 .../beam/dsls/sql/transform/package-info.java |  22 ++
 dsls/sql/src/main/resources/log4j.properties  |  23 ++
 .../beam/dsls/sql/planner/BasePlanner.java    |  74 ++++++
 .../sql/planner/BeamPlannerExplainTest.java   |  68 +++++
 .../sql/planner/BeamPlannerSubmitTest.java    |  42 +++
 .../dsls/sql/planner/MockedBeamSQLTable.java  | 123 +++++++++
 52 files changed, 3441 insertions(+), 2 deletions(-)
 create mode 100644 dsls/sql/README.md
 create mode 100644 dsls/sql/pom.xml
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/example/BeamSqlExample.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/example/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/CalciteToSpEL.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamPipelineCreator.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamQueryPlanner.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRuleSets.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSQLRelUtils.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlRunner.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/planner/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamFilterRel.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSinkRel.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSourceRel.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamLogicalConvention.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamProjectRel.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamRelNode.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rel/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamFilterRule.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSinkRule.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSourceRule.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamProjectRule.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/BaseBeamTable.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamIOType.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordType.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRow.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSqlRowCoder.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/InvalidFieldException.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/UnsupportedDataTypeException.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/schema/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLFilterFn.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLProjectFn.java
 create mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/transform/package-info.java
 create mode 100644 dsls/sql/src/main/resources/log4j.properties
 create mode 100644 dsls/sql/src/test/java/org/beam/dsls/sql/planner/BasePlanner.java
 create mode 100644 dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerExplainTest.java
 create mode 100644 dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
 create mode 100644 dsls/sql/src/test/java/org/beam/dsls/sql/planner/MockedBeamSQLTable.java

diff --git a/dsls/pom.xml b/dsls/pom.xml
index 6e0017115a41..a1bb0ee8b766 100644
--- a/dsls/pom.xml
+++ b/dsls/pom.xml
@@ -27,10 +27,11 @@
   </parent>
 
   <artifactId>beam-dsls-parent</artifactId>
+  <packaging>pom</packaging>
   <name>Apache Beam :: DSLs</name>
 
   <modules>
-    <!-- <module>sql</module> -->
+    <module>sql</module>
   </modules>
 
   <build>
@@ -53,4 +54,4 @@
     </pluginManagement>
   </build>
 
-</project>
\ No newline at end of file
+</project>
diff --git a/dsls/sql/README.md b/dsls/sql/README.md
new file mode 100644
index 000000000000..ae9e0f3f6fd5
--- /dev/null
+++ b/dsls/sql/README.md
@@ -0,0 +1,24 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
+# Beam SQL
+
+Beam SQL provides a new interface, to execute a SQL query as a Beam pipeline.
+
+*It's working in progress...*
diff --git a/dsls/sql/pom.xml b/dsls/sql/pom.xml
new file mode 100644
index 000000000000..21c8def16074
--- /dev/null
+++ b/dsls/sql/pom.xml
@@ -0,0 +1,166 @@
+<?xml version="1.0"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+  xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.beam</groupId>
+    <artifactId>beam-dsls-parent</artifactId>
+    <version>0.7.0-SNAPSHOT</version>
+  </parent>
+  
+  <artifactId>beam-dsls-sql</artifactId>
+  <name>Apache Beam :: DSLs :: SQL</name>
+  <description>Beam SQL provides a new interface to generate a Beam pipeline from SQL statement</description>
+
+  <packaging>jar</packaging>
+
+  <properties>
+    <timestamp>${maven.build.timestamp}</timestamp>
+    <maven.build.timestamp.format>yyyy-MM-dd HH:mm</maven.build.timestamp.format>
+    <calcite-version>1.11.0</calcite-version>
+  </properties>
+  
+  <build>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+        <filtering>true</filtering>
+      </resource>
+    </resources>
+
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-checkstyle-plugin</artifactId>
+          <configuration>
+            <!-- Set testSourceDirectory in order to exclude generated-test-sources -->
+            <testSourceDirectory>${project.basedir}/src/test/</testSourceDirectory>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+        <argLine>-da</argLine> <!-- disable assert in Calcite converter validation -->
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>bundle-and-repackage</id>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <shadeTestJar>true</shadeTestJar>
+              <artifactSet>
+                <includes>
+                  <include>com.google.guava:guava</include>
+                </includes>
+              </artifactSet>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+      <!-- Coverage analysis for unit tests. -->
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+  
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.calcite</groupId>
+      <artifactId>calcite-core</artifactId>
+      <version>${calcite-version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-runners-direct-java</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-io-kafka</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-expression</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-jdk14</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.calcite</groupId>
+      <artifactId>calcite-linq4j</artifactId>
+      <version>${calcite-version}</version>
+    </dependency>
+  </dependencies>
+</project>
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/example/BeamSqlExample.java b/dsls/sql/src/main/java/org/beam/dsls/sql/example/BeamSqlExample.java
new file mode 100644
index 000000000000..7fb8def5d76f
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/example/BeamSqlExample.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.example;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.beam.dsls.sql.planner.BeamSqlRunner;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
+
+/**
+ * This is one quick example.
+ * <p>Before start, follow https://kafka.apache.org/quickstart to setup a Kafka
+ * cluster locally, and run below commands to create required Kafka topics:
+ * <pre>
+ * <code>
+ * bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 \
+ *   --partitions 1 --topic orders
+ * bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 \
+ *   --partitions 1 --topic sub_orders
+ * </code>
+ * </pre>
+ * After run the application, produce several test records:
+ * <pre>
+ * <code>
+ * bin/kafka-console-producer.sh --broker-list localhost:9092 --topic orders
+ * invalid,record
+ * 123445,0,100,3413423
+ * 234123,3,232,3451231234
+ * 234234,0,5,1234123
+ * 345234,0,345234.345,3423
+ * </code>
+ * </pre>
+ * Meanwhile, open another console to see the output:
+ * <pre>
+ * <code>
+ * bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic sub_orders
+ * **Expected :
+ * 123445,0,100.0
+ * 345234,0,345234.345
+ * </code>
+ * </pre>
+ */
+public class BeamSqlExample implements Serializable {
+
+  /**
+   *
+   */
+  private static final long serialVersionUID = 3673487843555563904L;
+
+  public static void main(String[] args) throws Exception {
+    BeamSqlRunner runner = new BeamSqlRunner();
+    runner.addTable("ORDER_DETAILS", getTable("127.0.0.1:9092", "orders"));
+    runner.addTable("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
+
+    // case 2: insert into <table>(<fields>) select STREAM <fields> from
+    // <table> from <clause>
+    String sql = "INSERT INTO SUB_ORDER(order_id, site_id, price) " + "SELECT "
+        + " order_id, site_id, price " + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 and price > 20";
+
+    runner.explainQuery(sql);
+    runner.submitQuery(sql);
+  }
+
+  public static BaseBeamTable getTable(String bootstrapServer, String topic) {
+    final RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder().add("order_id", SqlTypeName.BIGINT).add("site_id", SqlTypeName.INTEGER)
+            .add("price", SqlTypeName.DOUBLE).add("order_time", SqlTypeName.TIMESTAMP).build();
+      }
+    };
+
+    Map<String, Object> consumerPara = new HashMap<String, Object>();
+    consumerPara.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
+
+    return new BeamKafkaCSVTable(protoRowType, bootstrapServer, Arrays.asList(topic))
+        .updateConsumerProperties(consumerPara);
+  }
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/example/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/example/package-info.java
new file mode 100644
index 000000000000..ae678e4297fd
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/example/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * examples on how to use BeamSQL.
+ *
+ */
+package org.beam.dsls.sql.example;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
new file mode 100644
index 000000000000..e9d425dd485a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.interpreter;
+
+import java.io.Serializable;
+import java.util.List;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ * {@code BeamSQLExpressionExecutor} fills the gap between relational
+ * expressions in Calcite SQL and executable code.
+ *
+ */
+public interface BeamSQLExpressionExecutor extends Serializable {
+
+  /**
+   * invoked before data processing.
+   */
+  void prepare();
+
+  /**
+   * apply transformation to input record {@link BeamSQLRow}.
+   *
+   */
+  List<Object> execute(BeamSQLRow inputRecord);
+
+  void close();
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
new file mode 100644
index 000000000000..48306da0dc91
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.interpreter;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.beam.dsls.sql.planner.BeamSqlUnsupportedException;
+import org.beam.dsls.sql.rel.BeamFilterRel;
+import org.beam.dsls.sql.rel.BeamProjectRel;
+import org.beam.dsls.sql.rel.BeamRelNode;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+import org.springframework.expression.Expression;
+import org.springframework.expression.ExpressionParser;
+import org.springframework.expression.spel.SpelParserConfiguration;
+import org.springframework.expression.spel.standard.SpelExpressionParser;
+import org.springframework.expression.spel.support.StandardEvaluationContext;
+
+/**
+ * {@code BeamSQLSpELExecutor} is one implementation, to convert Calcite SQL
+ * relational expression to SpEL expression.
+ *
+ */
+public class BeamSQLSpELExecutor implements BeamSQLExpressionExecutor {
+  /**
+   *
+   */
+  private static final long serialVersionUID = 6777232573390074408L;
+
+  private List<String> spelString;
+  private List<Expression> spelExpressions;
+
+  public BeamSQLSpELExecutor(BeamRelNode relNode) {
+    this.spelString = new ArrayList<>();
+    if (relNode instanceof BeamFilterRel) {
+      String filterSpEL = CalciteToSpEL
+          .rexcall2SpEL((RexCall) ((BeamFilterRel) relNode).getCondition());
+      spelString.add(filterSpEL);
+    } else if (relNode instanceof BeamProjectRel) {
+      spelString.addAll(createProjectExps((BeamProjectRel) relNode));
+      // List<ProjectRule> projectRules =
+      // for (int idx = 0; idx < projectRules.size(); ++idx) {
+      // spelString.add(projectRules.get(idx).getProjectExp());
+      // }
+    } else {
+      throw new BeamSqlUnsupportedException(
+          String.format("%s is not supported yet", relNode.getClass().toString()));
+    }
+  }
+
+  @Override
+  public void prepare() {
+    this.spelExpressions = new ArrayList<>();
+
+    SpelParserConfiguration config = new SpelParserConfiguration(true, true);
+    ExpressionParser parser = new SpelExpressionParser(config);
+    for (String el : spelString) {
+      spelExpressions.add(parser.parseExpression(el));
+    }
+  }
+
+  @Override
+  public List<Object> execute(BeamSQLRow inputRecord) {
+    StandardEvaluationContext inContext = new StandardEvaluationContext();
+    inContext.setVariable("in", inputRecord);
+
+    List<Object> results = new ArrayList<>();
+    for (Expression ep : spelExpressions) {
+      results.add(ep.getValue(inContext));
+    }
+    return results;
+  }
+
+  @Override
+  public void close() {
+
+  }
+
+  private List<String> createProjectExps(BeamProjectRel projectRel) {
+    List<String> rules = new ArrayList<>();
+
+    List<RexNode> exps = projectRel.getProjects();
+
+    for (int idx = 0; idx < exps.size(); ++idx) {
+      RexNode node = exps.get(idx);
+      if (node == null) {
+        rules.add("null");
+      }
+
+      if (node instanceof RexLiteral) {
+        rules.add(((RexLiteral) node).getValue() + "");
+      } else {
+        if (node instanceof RexInputRef) {
+          rules.add("#in.getFieldValue(" + ((RexInputRef) node).getIndex() + ")");
+        }
+        if (node instanceof RexCall) {
+          rules.add(CalciteToSpEL.rexcall2SpEL((RexCall) node));
+        }
+      }
+    }
+
+    checkArgument(rules.size() == exps.size(), "missing projects rules after conversion.");
+
+    return rules;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/CalciteToSpEL.java b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/CalciteToSpEL.java
new file mode 100644
index 000000000000..c7cbace4e360
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/CalciteToSpEL.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.interpreter;
+
+import com.google.common.base.Joiner;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.beam.dsls.sql.planner.BeamSqlUnsupportedException;
+
+/**
+ * {@code CalciteToSpEL} is used in {@link BeamSQLSpELExecutor}, to convert a
+ * relational expression {@link RexCall} to SpEL expression.
+ *
+ */
+public class CalciteToSpEL {
+
+  public static String rexcall2SpEL(RexCall cdn) {
+    List<String> parts = new ArrayList<>();
+    for (RexNode subcdn : cdn.operands) {
+      if (subcdn instanceof RexCall) {
+        parts.add(rexcall2SpEL((RexCall) subcdn));
+      } else {
+        parts.add(subcdn instanceof RexInputRef
+            ? "#in.getFieldValue(" + ((RexInputRef) subcdn).getIndex() + ")" : subcdn.toString());
+      }
+    }
+
+    String opName = cdn.op.getName();
+    switch (cdn.op.getClass().getSimpleName()) {
+    case "SqlMonotonicBinaryOperator": // +-*
+    case "SqlBinaryOperator": // > < = >= <= <> OR AND || / .
+      switch (cdn.op.getName().toUpperCase()) {
+      case "AND":
+        return String.format(" ( %s ) ", Joiner.on("&&").join(parts));
+      case "OR":
+        return String.format(" ( %s ) ", Joiner.on("||").join(parts));
+      case "=":
+        return String.format(" ( %s ) ", Joiner.on("==").join(parts));
+      case "<>":
+        return String.format(" ( %s ) ", Joiner.on("!=").join(parts));
+      default:
+        return String.format(" ( %s ) ", Joiner.on(cdn.op.getName().toUpperCase()).join(parts));
+      }
+    case "SqlCaseOperator": // CASE
+      return String.format(" (%s ? %s : %s)", parts.get(0), parts.get(1), parts.get(2));
+    case "SqlCastFunction": // CAST
+      return parts.get(0);
+    case "SqlPostfixOperator":
+      switch (opName.toUpperCase()) {
+      case "IS NULL":
+        return String.format(" null == %s ", parts.get(0));
+      case "IS NOT NULL":
+        return String.format(" null != %s ", parts.get(0));
+      default:
+        throw new BeamSqlUnsupportedException();
+      }
+    default:
+      throw new BeamSqlUnsupportedException();
+    }
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/package-info.java
new file mode 100644
index 000000000000..85235e2dcb0b
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * interpreter generate runnable 'code' to execute SQL relational expressions.
+ */
+package org.beam.dsls.sql.interpreter;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/package-info.java
new file mode 100644
index 000000000000..c6f5cf63ba8e
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * BeamSQL provides a new interface to run a SQL statement with Beam.
+ */
+package org.beam.dsls.sql;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamPipelineCreator.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamPipelineCreator.java
new file mode 100644
index 000000000000..5a0c73d2a335
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamPipelineCreator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.values.PCollection;
+import org.beam.dsls.sql.rel.BeamRelNode;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.beam.dsls.sql.schema.BeamSQLRecordTypeCoder;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+import org.beam.dsls.sql.schema.BeamSqlRowCoder;
+
+/**
+ * {@link BeamPipelineCreator} converts a {@link BeamRelNode} tree, into a Beam
+ * pipeline.
+ *
+ */
+public class BeamPipelineCreator {
+  private Map<String, BaseBeamTable> sourceTables;
+  private PCollection<BeamSQLRow> latestStream;
+
+  private PipelineOptions options;
+
+  private Pipeline pipeline;
+
+  private boolean hasPersistent = false;
+
+  public BeamPipelineCreator(Map<String, BaseBeamTable> sourceTables) {
+    this.sourceTables = sourceTables;
+
+    options = PipelineOptionsFactory.fromArgs(new String[] {}).withValidation()
+        .as(PipelineOptions.class); // FlinkPipelineOptions.class
+    options.setJobName("BeamPlanCreator");
+
+    pipeline = Pipeline.create(options);
+    CoderRegistry cr = pipeline.getCoderRegistry();
+    cr.registerCoder(BeamSQLRow.class, BeamSqlRowCoder.of());
+    cr.registerCoder(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
+  }
+
+  public PCollection<BeamSQLRow> getLatestStream() {
+    return latestStream;
+  }
+
+  public void setLatestStream(PCollection<BeamSQLRow> latestStream) {
+    this.latestStream = latestStream;
+  }
+
+  public Map<String, BaseBeamTable> getSourceTables() {
+    return sourceTables;
+  }
+
+  public Pipeline getPipeline() {
+    return pipeline;
+  }
+
+  public boolean isHasPersistent() {
+    return hasPersistent;
+  }
+
+  public void setHasPersistent(boolean hasPersistent) {
+    this.hasPersistent = hasPersistent;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamQueryPlanner.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamQueryPlanner.java
new file mode 100644
index 000000000000..a31ace0a584d
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamQueryPlanner.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalciteCatalogReader;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.beam.dsls.sql.rel.BeamLogicalConvention;
+import org.beam.dsls.sql.rel.BeamRelNode;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The core component to handle through a SQL statement, to submit a Beam
+ * pipeline.
+ *
+ */
+public class BeamQueryPlanner {
+  private static final Logger LOG = LoggerFactory.getLogger(BeamQueryPlanner.class);
+
+  protected final Planner planner;
+  private Map<String, BaseBeamTable> sourceTables = new HashMap<>();
+
+  public static final JavaTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(
+      RelDataTypeSystem.DEFAULT);
+
+  public BeamQueryPlanner(SchemaPlus schema) {
+    final List<RelTraitDef> traitDefs = new ArrayList<RelTraitDef>();
+    traitDefs.add(ConventionTraitDef.INSTANCE);
+    traitDefs.add(RelCollationTraitDef.INSTANCE);
+
+    List<SqlOperatorTable> sqlOperatorTables = new ArrayList<>();
+    sqlOperatorTables.add(SqlStdOperatorTable.instance());
+    sqlOperatorTables.add(new CalciteCatalogReader(CalciteSchema.from(schema), false,
+        Collections.<String>emptyList(), TYPE_FACTORY));
+
+    FrameworkConfig config = Frameworks.newConfigBuilder()
+        .parserConfig(SqlParser.configBuilder().setLex(Lex.MYSQL).build()).defaultSchema(schema)
+        .traitDefs(traitDefs).context(Contexts.EMPTY_CONTEXT).ruleSets(BeamRuleSets.getRuleSets())
+        .costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM).build();
+    this.planner = Frameworks.getPlanner(config);
+
+    for (String t : schema.getTableNames()) {
+      sourceTables.put(t, (BaseBeamTable) schema.getTable(t));
+    }
+  }
+
+  /**
+   * With a Beam pipeline generated in {@link #compileBeamPipeline(String)},
+   * submit it to run and wait until finish.
+   *
+   */
+  public void submitToRun(String sqlStatement) throws Exception {
+    Pipeline pipeline = compileBeamPipeline(sqlStatement);
+
+    PipelineResult result = pipeline.run();
+    result.waitUntilFinish();
+  }
+
+  /**
+   * With the @{@link BeamRelNode} tree generated in
+   * {@link #convertToBeamRel(String)}, a Beam pipeline is generated.
+   *
+   */
+  public Pipeline compileBeamPipeline(String sqlStatement) throws Exception {
+    BeamRelNode relNode = convertToBeamRel(sqlStatement);
+
+    BeamPipelineCreator planCreator = new BeamPipelineCreator(sourceTables);
+    return relNode.buildBeamPipeline(planCreator);
+  }
+
+  /**
+   * It parses and validate the input query, then convert into a
+   * {@link BeamRelNode} tree.
+   *
+   */
+  public BeamRelNode convertToBeamRel(String sqlStatement)
+      throws ValidationException, RelConversionException, SqlParseException {
+    return (BeamRelNode) validateAndConvert(planner.parse(sqlStatement));
+  }
+
+  private RelNode validateAndConvert(SqlNode sqlNode)
+      throws ValidationException, RelConversionException {
+    SqlNode validated = validateNode(sqlNode);
+    LOG.info("SQL:\n" + validated);
+    RelNode relNode = convertToRelNode(validated);
+    return convertToBeamRel(relNode);
+  }
+
+  private RelNode convertToBeamRel(RelNode relNode) throws RelConversionException {
+    RelTraitSet traitSet = relNode.getTraitSet();
+
+    LOG.info("SQLPlan>\n" + RelOptUtil.toString(relNode));
+
+    // PlannerImpl.transform() optimizes RelNode with ruleset
+    return planner.transform(0, traitSet.plus(BeamLogicalConvention.INSTANCE), relNode);
+  }
+
+  private RelNode convertToRelNode(SqlNode sqlNode) throws RelConversionException {
+    return planner.rel(sqlNode).rel;
+  }
+
+  private SqlNode validateNode(SqlNode sqlNode) throws ValidationException {
+    SqlNode validatedSqlNode = planner.validate(sqlNode);
+    validatedSqlNode.accept(new UnsupportedOperatorsVisitor());
+    return validatedSqlNode;
+  }
+
+  public Map<String, BaseBeamTable> getSourceTables() {
+    return sourceTables;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRelDataTypeSystem.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
new file mode 100644
index 000000000000..bf35296df648
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rel.type.RelDataTypeSystemImpl;
+
+/**
+ * customized data type in Beam.
+ *
+ */
+public class BeamRelDataTypeSystem extends RelDataTypeSystemImpl {
+  public static final RelDataTypeSystem BEAM_REL_DATATYPE_SYSTEM = new BeamRelDataTypeSystem();
+
+  @Override
+  public int getMaxNumericScale() {
+    return 38;
+  }
+
+  @Override
+  public int getMaxNumericPrecision() {
+    return 38;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRuleSets.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRuleSets.java
new file mode 100644
index 000000000000..3f40c271c486
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRuleSets.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import java.util.Iterator;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.RuleSet;
+import org.beam.dsls.sql.rel.BeamRelNode;
+import org.beam.dsls.sql.rule.BeamFilterRule;
+import org.beam.dsls.sql.rule.BeamIOSinkRule;
+import org.beam.dsls.sql.rule.BeamIOSourceRule;
+import org.beam.dsls.sql.rule.BeamProjectRule;
+
+/**
+ * {@link RuleSet} used in {@link BeamQueryPlanner}. It translates a standard
+ * Calcite {@link RelNode} tree, to represent with {@link BeamRelNode}
+ *
+ */
+public class BeamRuleSets {
+  private static final ImmutableSet<RelOptRule> calciteToBeamConversionRules = ImmutableSet
+      .<RelOptRule>builder().add(BeamIOSourceRule.INSTANCE, BeamProjectRule.INSTANCE,
+          BeamFilterRule.INSTANCE, BeamIOSinkRule.INSTANCE)
+      .build();
+
+  public static RuleSet[] getRuleSets() {
+    return new RuleSet[] { new BeamRuleSet(
+        ImmutableSet.<RelOptRule>builder().addAll(calciteToBeamConversionRules).build()) };
+  }
+
+  private static class BeamRuleSet implements RuleSet {
+    final ImmutableSet<RelOptRule> rules;
+
+    public BeamRuleSet(ImmutableSet<RelOptRule> rules) {
+      this.rules = rules;
+    }
+
+    public BeamRuleSet(ImmutableList<RelOptRule> rules) {
+      this.rules = ImmutableSet.<RelOptRule>builder().addAll(rules).build();
+    }
+
+    @Override
+    public Iterator<RelOptRule> iterator() {
+      return rules.iterator();
+    }
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSQLRelUtils.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSQLRelUtils.java
new file mode 100644
index 000000000000..94b341c82dba
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSQLRelUtils.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.beam.dsls.sql.rel.BeamRelNode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utilities for {@code BeamRelNode}.
+ */
+public class BeamSQLRelUtils {
+  private static final Logger LOG = LoggerFactory.getLogger(BeamSQLRelUtils.class);
+
+  private static final AtomicInteger sequence = new AtomicInteger(0);
+  private static final AtomicInteger classSequence = new AtomicInteger(0);
+
+  public static String getStageName(BeamRelNode relNode) {
+    return relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId() + "_"
+        + sequence.getAndIncrement();
+  }
+
+  public static String getClassName(BeamRelNode relNode) {
+    return "Generated_" + relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId()
+        + "_" + classSequence.getAndIncrement();
+  }
+
+  public static BeamRelNode getBeamRelInput(RelNode input) {
+    if (input instanceof RelSubset) {
+      // go with known best input
+      input = ((RelSubset) input).getBest();
+    }
+    return (BeamRelNode) input;
+  }
+
+  public static String explain(final RelNode rel) {
+    return explain(rel, SqlExplainLevel.EXPPLAN_ATTRIBUTES);
+  }
+
+  public static String explain(final RelNode rel, SqlExplainLevel detailLevel) {
+    String explain = "";
+    try {
+      explain = RelOptUtil.toString(rel);
+    } catch (StackOverflowError e) {
+      LOG.error("StackOverflowError occurred while extracting plan. "
+          + "Please report it to the dev@ mailing list.");
+      LOG.error("RelNode " + rel + " ExplainLevel " + detailLevel, e);
+      LOG.error("Forcing plan to empty string and continue... "
+          + "SQL Runner may not working properly after.");
+    }
+    return explain;
+  }
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlRunner.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlRunner.java
new file mode 100644
index 000000000000..9581fcdec50d
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlRunner.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import java.io.Serializable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.beam.dsls.sql.rel.BeamRelNode;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Interface to explain, submit a SQL query.
+ *
+ */
+public class BeamSqlRunner implements Serializable {
+  /**
+   *
+   */
+  private static final long serialVersionUID = -4708693435115005182L;
+
+  private static final Logger LOG = LoggerFactory.getLogger(BeamSqlRunner.class);
+
+  private SchemaPlus schema = Frameworks.createRootSchema(true);
+
+  private BeamQueryPlanner planner = new BeamQueryPlanner(schema);
+
+  /**
+   * Add a schema.
+   *
+   */
+  public void addSchema(String schemaName, Schema scheme) {
+    schema.add(schemaName, schema);
+  }
+
+  /**
+   * add a {@link BaseBeamTable} to schema repository.
+   *
+   */
+  public void addTable(String tableName, BaseBeamTable table) {
+    schema.add(tableName, table);
+    planner.getSourceTables().put(tableName, table);
+  }
+
+  /**
+   * submit as a Beam pipeline.
+   *
+   */
+  public void submitQuery(String sqlString) throws Exception {
+    planner.submitToRun(sqlString);
+    planner.planner.close();
+  }
+
+  /**
+   * explain and display the execution plan.
+   *
+   */
+  public String explainQuery(String sqlString)
+      throws ValidationException, RelConversionException, SqlParseException {
+    BeamRelNode exeTree = planner.convertToBeamRel(sqlString);
+    String beamPlan = RelOptUtil.toString(exeTree);
+    System.out.println(String.format("beamPlan>\n%s", beamPlan));
+
+    planner.planner.close();
+    return beamPlan;
+  }
+
+  protected BeamQueryPlanner getPlanner() {
+    return planner;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlUnsupportedException.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
new file mode 100644
index 000000000000..a3475bb1c261
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+/**
+ * Generic exception for un-supported operations.
+ *
+ */
+public class BeamSqlUnsupportedException extends RuntimeException {
+  /**
+   *
+   */
+  private static final long serialVersionUID = 3445015747629217342L;
+
+  public BeamSqlUnsupportedException(String string) {
+    super(string);
+  }
+
+  public BeamSqlUnsupportedException() {
+    super();
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
new file mode 100644
index 000000000000..702381df60be
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import org.apache.calcite.sql.util.SqlShuttle;
+
+/**
+ * Unsupported operation to visit a RelNode.
+ *
+ */
+public class UnsupportedOperatorsVisitor extends SqlShuttle {
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/package-info.java
new file mode 100644
index 000000000000..d98c58424121
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/planner/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.beam.dsls.sql.planner.BeamQueryPlanner} is the main interface.
+ * It defines data sources, validate a SQL statement, and convert it as a Beam
+ * pipeline.
+ */
+package org.beam.dsls.sql.planner;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamFilterRel.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamFilterRel.java
new file mode 100644
index 000000000000..64f2d1fa6700
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamFilterRel.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rel;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rex.RexNode;
+import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
+import org.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+import org.beam.dsls.sql.transform.BeamSQLFilterFn;
+
+/**
+ * BeamRelNode to replace a {@code Filter} node.
+ *
+ */
+public class BeamFilterRel extends Filter implements BeamRelNode {
+
+  public BeamFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child,
+      RexNode condition) {
+    super(cluster, traits, child, condition);
+  }
+
+  @Override
+  public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
+    return new BeamFilterRel(getCluster(), traitSet, input, condition);
+  }
+
+  @Override
+  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+
+    RelNode input = getInput();
+    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
+
+    String stageName = BeamSQLRelUtils.getStageName(this);
+
+    PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
+
+    BeamSQLExpressionExecutor executor = new BeamSQLSpELExecutor(this);
+
+    PCollection<BeamSQLRow> projectStream = upstream.apply(stageName,
+        ParDo.of(new BeamSQLFilterFn(getRelTypeName(), executor)));
+
+    planCreator.setLatestStream(projectStream);
+
+    return planCreator.getPipeline();
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSinkRel.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSinkRel.java
new file mode 100644
index 000000000000..46654e5e372a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSinkRel.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rel;
+
+import com.google.common.base.Joiner;
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rex.RexNode;
+import org.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ * BeamRelNode to replace a {@code TableModify} node.
+ *
+ */
+public class BeamIOSinkRel extends TableModify implements BeamRelNode {
+  public BeamIOSinkRel(RelOptCluster cluster, RelTraitSet traits, RelOptTable table,
+      Prepare.CatalogReader catalogReader, RelNode child, Operation operation,
+      List<String> updateColumnList, List<RexNode> sourceExpressionList, boolean flattened) {
+    super(cluster, traits, table, catalogReader, child, operation, updateColumnList,
+        sourceExpressionList, flattened);
+  }
+
+  @Override
+  public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+    return new BeamIOSinkRel(getCluster(), traitSet, getTable(), getCatalogReader(), sole(inputs),
+        getOperation(), getUpdateColumnList(), getSourceExpressionList(), isFlattened());
+  }
+
+  @Override
+  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+
+    RelNode input = getInput();
+    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
+
+    String stageName = BeamSQLRelUtils.getStageName(this);
+
+    PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
+
+    String sourceName = Joiner.on('.').join(getTable().getQualifiedName());
+
+    BaseBeamTable targetTable = planCreator.getSourceTables().get(sourceName);
+
+    upstream.apply(stageName, targetTable.buildIOWriter());
+
+    planCreator.setHasPersistent(true);
+
+    return planCreator.getPipeline();
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSourceRel.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSourceRel.java
new file mode 100644
index 000000000000..f14db922e63e
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSourceRel.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rel;
+
+import com.google.common.base.Joiner;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.core.TableScan;
+import org.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ * BeamRelNode to replace a {@code TableScan} node.
+ *
+ */
+public class BeamIOSourceRel extends TableScan implements BeamRelNode {
+
+  public BeamIOSourceRel(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table) {
+    super(cluster, traitSet, table);
+  }
+
+  @Override
+  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+
+    String sourceName = Joiner.on('.').join(getTable().getQualifiedName()).replace(".(STREAM)", "");
+
+    BaseBeamTable sourceTable = planCreator.getSourceTables().get(sourceName);
+
+    String stageName = BeamSQLRelUtils.getStageName(this);
+
+    PCollection<BeamSQLRow> sourceStream = planCreator.getPipeline().apply(stageName,
+        sourceTable.buildIOReader());
+
+    planCreator.setLatestStream(sourceStream);
+
+    return planCreator.getPipeline();
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamLogicalConvention.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamLogicalConvention.java
new file mode 100644
index 000000000000..50fe8e013af1
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamLogicalConvention.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rel;
+
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.RelTraitSet;
+
+/**
+ * Convertion for Beam SQL.
+ *
+ */
+public enum BeamLogicalConvention implements Convention {
+  INSTANCE;
+
+  @Override
+  public Class getInterface() {
+    return BeamRelNode.class;
+  }
+
+  @Override
+  public String getName() {
+    return "BEAM_LOGICAL";
+  }
+
+  @Override
+  public RelTraitDef getTraitDef() {
+    return ConventionTraitDef.INSTANCE;
+  }
+
+  @Override
+  public boolean satisfies(RelTrait trait) {
+    return this == trait;
+  }
+
+  @Override
+  public void register(RelOptPlanner planner) {
+  }
+
+  @Override
+  public String toString() {
+    return getName();
+  }
+
+  @Override
+  public boolean canConvertConvention(Convention toConvention) {
+    return false;
+  }
+
+  @Override
+  public boolean useAbstractConvertersForConversion(RelTraitSet fromTraits, RelTraitSet toTraits) {
+    return false;
+  }
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamProjectRel.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamProjectRel.java
new file mode 100644
index 000000000000..e41d74ecd37c
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamProjectRel.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rel;
+
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
+import org.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+import org.beam.dsls.sql.transform.BeamSQLProjectFn;
+
+/**
+ * BeamRelNode to replace a {@code Project} node.
+ *
+ */
+public class BeamProjectRel extends Project implements BeamRelNode {
+
+  /**
+   * projects: {@link RexLiteral}, {@link RexInputRef}, {@link RexCall}.
+   *
+   */
+  public BeamProjectRel(RelOptCluster cluster, RelTraitSet traits, RelNode input,
+      List<? extends RexNode> projects, RelDataType rowType) {
+    super(cluster, traits, input, projects, rowType);
+  }
+
+  @Override
+  public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> projects,
+      RelDataType rowType) {
+    return new BeamProjectRel(getCluster(), traitSet, input, projects, rowType);
+  }
+
+  @Override
+  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+    RelNode input = getInput();
+    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
+
+    String stageName = BeamSQLRelUtils.getStageName(this);
+
+    PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
+
+    BeamSQLExpressionExecutor executor = new BeamSQLSpELExecutor(this);
+
+    PCollection<BeamSQLRow> projectStream = upstream.apply(stageName, ParDo
+        .of(new BeamSQLProjectFn(getRelTypeName(), executor, BeamSQLRecordType.from(rowType))));
+
+    planCreator.setLatestStream(projectStream);
+
+    return planCreator.getPipeline();
+
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamRelNode.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamRelNode.java
new file mode 100644
index 000000000000..07ffee50237c
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamRelNode.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rel;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.calcite.rel.RelNode;
+import org.beam.dsls.sql.planner.BeamPipelineCreator;
+
+/**
+ * A new method {@link #buildBeamPipeline(BeamPipelineCreator)} is added, it's
+ * called by {@link BeamPipelineCreator}.
+ *
+ */
+public interface BeamRelNode extends RelNode {
+
+  /**
+   * A {@link BeamRelNode} is a recursive structure, the
+   * {@link BeamPipelineCreator} visits it with a DFS(Depth-First-Search)
+   * algorithm.
+   *
+   */
+  Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception;
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/package-info.java
new file mode 100644
index 000000000000..13dc96285942
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rel/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * BeamSQL specified nodes, to replace {@link org.apache.calcite.rel.RelNode}.
+ *
+ */
+package org.beam.dsls.sql.rel;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamFilterRule.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamFilterRule.java
new file mode 100644
index 000000000000..2ad7c074dbdc
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamFilterRule.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rule;
+
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.logical.LogicalFilter;
+import org.beam.dsls.sql.rel.BeamFilterRel;
+import org.beam.dsls.sql.rel.BeamLogicalConvention;
+
+/**
+ * A {@code ConverterRule} to replace {@link Filter} with {@link BeamFilterRel}.
+ *
+ */
+public class BeamFilterRule extends ConverterRule {
+  public static final BeamFilterRule INSTANCE = new BeamFilterRule();
+
+  private BeamFilterRule() {
+    super(LogicalFilter.class, Convention.NONE, BeamLogicalConvention.INSTANCE, "BeamFilterRule");
+  }
+
+  @Override
+  public RelNode convert(RelNode rel) {
+    final Filter filter = (Filter) rel;
+    final RelNode input = filter.getInput();
+
+    return new BeamFilterRel(filter.getCluster(),
+        filter.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+        convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+        filter.getCondition());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSinkRule.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSinkRule.java
new file mode 100644
index 000000000000..a44c002f05c0
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSinkRule.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rule;
+
+import java.util.List;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rel.logical.LogicalTableModify;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.Table;
+import org.beam.dsls.sql.rel.BeamIOSinkRel;
+import org.beam.dsls.sql.rel.BeamLogicalConvention;
+
+/**
+ * A {@code ConverterRule} to replace {@link TableModify} with
+ * {@link BeamIOSinkRel}.
+ *
+ */
+public class BeamIOSinkRule extends ConverterRule {
+  public static final BeamIOSinkRule INSTANCE = new BeamIOSinkRule();
+
+  private BeamIOSinkRule() {
+    super(LogicalTableModify.class, Convention.NONE, BeamLogicalConvention.INSTANCE,
+        "BeamIOSinkRule");
+  }
+
+  @Override
+  public RelNode convert(RelNode rel) {
+    final TableModify tableModify = (TableModify) rel;
+    final RelNode input = tableModify.getInput();
+
+    final RelOptCluster cluster = tableModify.getCluster();
+    final RelTraitSet traitSet = tableModify.getTraitSet().replace(BeamLogicalConvention.INSTANCE);
+    final RelOptTable relOptTable = tableModify.getTable();
+    final Prepare.CatalogReader catalogReader = tableModify.getCatalogReader();
+    final RelNode convertedInput = convert(input,
+        input.getTraitSet().replace(BeamLogicalConvention.INSTANCE));
+    final TableModify.Operation operation = tableModify.getOperation();
+    final List<String> updateColumnList = tableModify.getUpdateColumnList();
+    final List<RexNode> sourceExpressionList = tableModify.getSourceExpressionList();
+    final boolean flattened = tableModify.isFlattened();
+
+    final Table table = tableModify.getTable().unwrap(Table.class);
+
+    switch (table.getJdbcTableType()) {
+    case TABLE:
+    case STREAM:
+      if (operation != TableModify.Operation.INSERT) {
+        throw new UnsupportedOperationException(
+            String.format("Streams doesn't support %s modify operation", operation));
+      }
+      return new BeamIOSinkRel(cluster, traitSet,
+          relOptTable, catalogReader, convertedInput, operation, updateColumnList,
+          sourceExpressionList, flattened);
+    default:
+      throw new IllegalArgumentException(
+          String.format("Unsupported table type: %s", table.getJdbcTableType()));
+    }
+  }
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSourceRule.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSourceRule.java
new file mode 100644
index 000000000000..9e4778b5ff04
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSourceRule.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rule;
+
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.logical.LogicalTableScan;
+import org.beam.dsls.sql.rel.BeamIOSourceRel;
+import org.beam.dsls.sql.rel.BeamLogicalConvention;
+
+/**
+ * A {@code ConverterRule} to replace {@link TableScan} with
+ * {@link BeamIOSourceRel}.
+ *
+ */
+public class BeamIOSourceRule extends ConverterRule {
+  public static final BeamIOSourceRule INSTANCE = new BeamIOSourceRule();
+
+  private BeamIOSourceRule() {
+    super(LogicalTableScan.class, Convention.NONE, BeamLogicalConvention.INSTANCE,
+        "BeamIOSourceRule");
+  }
+
+  @Override
+  public RelNode convert(RelNode rel) {
+    final TableScan scan = (TableScan) rel;
+
+    return new BeamIOSourceRel(scan.getCluster(),
+        scan.getTraitSet().replace(BeamLogicalConvention.INSTANCE), scan.getTable());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamProjectRule.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamProjectRule.java
new file mode 100644
index 000000000000..117a056fa0ff
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamProjectRule.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.rule;
+
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.beam.dsls.sql.rel.BeamLogicalConvention;
+import org.beam.dsls.sql.rel.BeamProjectRel;
+
+/**
+ * A {@code ConverterRule} to replace {@link Project} with
+ * {@link BeamProjectRel}.
+ *
+ */
+public class BeamProjectRule extends ConverterRule {
+  public static final BeamProjectRule INSTANCE = new BeamProjectRule();
+
+  private BeamProjectRule() {
+    super(LogicalProject.class, Convention.NONE, BeamLogicalConvention.INSTANCE, "BeamProjectRule");
+  }
+
+  @Override
+  public RelNode convert(RelNode rel) {
+    final Project project = (Project) rel;
+    final RelNode input = project.getInput();
+
+    return new BeamProjectRel(project.getCluster(),
+        project.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+        convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+        project.getProjects(), project.getRowType());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java
new file mode 100644
index 000000000000..56ddcf3241b9
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.apache.calcite.plan.RelOptRule} to generate {@link org.beam.dsls.sql.rel.BeamRelNode}.
+ */
+package org.beam.dsls.sql.rule;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BaseBeamTable.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BaseBeamTable.java
new file mode 100644
index 000000000000..3816063ab297
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BaseBeamTable.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.calcite.DataContext;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.schema.ScannableTable;
+import org.apache.calcite.schema.Schema.TableType;
+import org.apache.calcite.schema.Statistic;
+import org.apache.calcite.schema.Statistics;
+import org.beam.dsls.sql.planner.BeamQueryPlanner;
+
+/**
+ * Each IO in Beam has one table schema, by extending {@link BaseBeamTable}.
+ */
+public abstract class BaseBeamTable implements ScannableTable, Serializable {
+
+  /**
+   *
+   */
+  private static final long serialVersionUID = -1262988061830914193L;
+  private RelDataType relDataType;
+
+  protected BeamSQLRecordType beamSqlRecordType;
+
+  public BaseBeamTable(RelProtoDataType protoRowType) {
+    this.relDataType = protoRowType.apply(BeamQueryPlanner.TYPE_FACTORY);
+    this.beamSqlRecordType = BeamSQLRecordType.from(relDataType);
+  }
+
+  /**
+   * In Beam SQL, there's no difference between a batch query and a streaming
+   * query. {@link BeamIOType} is used to validate the sources.
+   */
+  public abstract BeamIOType getSourceType();
+
+  /**
+   * create a {@code IO.read()} instance to read from source.
+   *
+   */
+  public abstract PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader();
+
+  /**
+   * create a {@code IO.write()} instance to write to target.
+   *
+   */
+  public abstract PTransform<? super PCollection<BeamSQLRow>, PDone> buildIOWriter();
+
+  @Override
+  public Enumerable<Object[]> scan(DataContext root) {
+    // not used as Beam SQL uses its own execution engine
+    return null;
+  }
+
+  @Override
+  public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+    return relDataType;
+  }
+
+  /**
+   * Not used {@link Statistic} to optimize the plan.
+   */
+  @Override
+  public Statistic getStatistic() {
+    return Statistics.UNKNOWN;
+  }
+
+  /**
+   * all sources are treated as TABLE in Beam SQL.
+   */
+  @Override
+  public TableType getJdbcTableType() {
+    return TableType.TABLE;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamIOType.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamIOType.java
new file mode 100644
index 000000000000..5e55b0fc6e48
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamIOType.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+import java.io.Serializable;
+
+/**
+ * Type as a source IO, determined whether it's a STREAMING process, or batch
+ * process.
+ */
+public enum BeamIOType implements Serializable {
+  BOUNDED, UNBOUNDED;
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordType.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordType.java
new file mode 100644
index 000000000000..dc8e38103cc6
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordType.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Field type information in {@link BeamSQLRow}.
+ *
+ */
+//@DefaultCoder(BeamSQLRecordTypeCoder.class)
+public class BeamSQLRecordType implements Serializable {
+  /**
+   *
+   */
+  private static final long serialVersionUID = -5318734648766104712L;
+  private List<String> fieldsName = new ArrayList<>();
+  private List<SqlTypeName> fieldsType = new ArrayList<>();
+
+  public static BeamSQLRecordType from(RelDataType tableInfo) {
+    BeamSQLRecordType record = new BeamSQLRecordType();
+    for (RelDataTypeField f : tableInfo.getFieldList()) {
+      record.fieldsName.add(f.getName());
+      record.fieldsType.add(f.getType().getSqlTypeName());
+    }
+    return record;
+  }
+
+  public int size() {
+    return fieldsName.size();
+  }
+
+  public List<String> getFieldsName() {
+    return fieldsName;
+  }
+
+  public void setFieldsName(List<String> fieldsName) {
+    this.fieldsName = fieldsName;
+  }
+
+  public List<SqlTypeName> getFieldsType() {
+    return fieldsType;
+  }
+
+  public void setFieldsType(List<SqlTypeName> fieldsType) {
+    this.fieldsType = fieldsType;
+  }
+
+  @Override
+  public String toString() {
+    return "RecordType [fieldsName=" + fieldsName + ", fieldsType=" + fieldsType + "]";
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
new file mode 100644
index 000000000000..2989cb93f4a4
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StandardCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * A {@link Coder} for {@link BeamSQLRecordType}.
+ *
+ */
+public class BeamSQLRecordTypeCoder extends StandardCoder<BeamSQLRecordType> {
+  private static final StringUtf8Coder stringCoder = StringUtf8Coder.of();
+  private static final VarIntCoder intCoder = VarIntCoder.of();
+
+  private static final BeamSQLRecordTypeCoder INSTANCE = new BeamSQLRecordTypeCoder();
+  private BeamSQLRecordTypeCoder(){}
+
+  public static BeamSQLRecordTypeCoder of() {
+    return INSTANCE;
+  }
+
+  @Override
+  public void encode(BeamSQLRecordType value, OutputStream outStream,
+      org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
+    Context nested = context.nested();
+    intCoder.encode(value.size(), outStream, nested);
+    for(String fieldName : value.getFieldsName()){
+      stringCoder.encode(fieldName, outStream, nested);
+    }
+    for(SqlTypeName fieldType : value.getFieldsType()){
+      stringCoder.encode(fieldType.name(), outStream, nested);
+    }
+    outStream.flush();
+  }
+
+  @Override
+  public BeamSQLRecordType decode(InputStream inStream,
+      org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
+    BeamSQLRecordType typeRecord = new BeamSQLRecordType();
+    Context nested = context.nested();
+    int size = intCoder.decode(inStream, nested);
+    for(int idx=0; idx<size; ++idx){
+      typeRecord.getFieldsName().add(stringCoder.decode(inStream, nested));
+    }
+    for(int idx=0; idx<size; ++idx){
+      typeRecord.getFieldsType().add(SqlTypeName.valueOf(stringCoder.decode(inStream, nested)));
+    }
+    return typeRecord;
+  }
+
+  @Override
+  public List<? extends Coder<?>> getCoderArguments() {
+    // TODO Auto-generated method stub
+    return null;
+  }
+
+  @Override
+  public void verifyDeterministic()
+      throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
+    // TODO Auto-generated method stub
+
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRow.java
new file mode 100644
index 000000000000..db931688122a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRow.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Repersent a generic ROW record in Beam SQL.
+ *
+ */
+public class BeamSQLRow implements Serializable {
+  /**
+   *
+   */
+  private static final long serialVersionUID = 4569220242480160895L;
+
+  private List<Integer> nullFields = new ArrayList<>();
+  private List<Object> dataValues;
+  private BeamSQLRecordType dataType;
+
+  public BeamSQLRow(BeamSQLRecordType dataType) {
+    this.dataType = dataType;
+    this.dataValues = new ArrayList<>();
+    for(int idx=0; idx<dataType.size(); ++idx){
+      dataValues.add(null);
+    }
+  }
+
+  public BeamSQLRow(BeamSQLRecordType dataType, List<Object> dataValues) {
+    this.dataValues = dataValues;
+    this.dataType = dataType;
+  }
+
+  public void addField(String fieldName, Object fieldValue) {
+    addField(dataType.getFieldsName().indexOf(fieldName), fieldValue);
+  }
+
+  public void addField(int index, Object fieldValue) {
+    if(fieldValue == null){
+      dataValues.set(index, fieldValue);
+      if(!nullFields.contains(index)){nullFields.add(index);}
+      return;
+    }
+
+    SqlTypeName fieldType = dataType.getFieldsType().get(index);
+    switch (fieldType) {
+    case INTEGER:
+    case SMALLINT:
+    case TINYINT:
+      if(!(fieldValue instanceof Integer)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
+    case DOUBLE:
+      if(!(fieldValue instanceof Double)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
+    case BIGINT:
+      if(!(fieldValue instanceof Long)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
+    case FLOAT:
+      if(!(fieldValue instanceof Float)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
+    case VARCHAR:
+      if(!(fieldValue instanceof String)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
+    case TIME:
+    case TIMESTAMP:
+      if(!(fieldValue instanceof Date)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
+    default:
+      throw new UnsupportedDataTypeException(fieldType);
+    }
+    dataValues.set(index, fieldValue);
+  }
+
+
+  public int getInteger(int idx) {
+    return (Integer) getFieldValue(idx);
+  }
+
+  public double getDouble(int idx) {
+    return (Double) getFieldValue(idx);
+  }
+
+  public long getLong(int idx) {
+    return (Long) getFieldValue(idx);
+  }
+
+  public String getString(int idx) {
+    return (String) getFieldValue(idx);
+  }
+
+  public Date getDate(int idx) {
+    return (Date) getFieldValue(idx);
+  }
+
+  public Object getFieldValue(String fieldName) {
+    return getFieldValue(dataType.getFieldsName().indexOf(fieldName));
+  }
+
+  public Object getFieldValue(int fieldIdx) {
+    if(nullFields.contains(fieldIdx)){
+      return null;
+    }
+
+    Object fieldValue = dataValues.get(fieldIdx);
+    SqlTypeName fieldType = dataType.getFieldsType().get(fieldIdx);
+
+    switch (fieldType) {
+    case INTEGER:
+    case SMALLINT:
+    case TINYINT:
+      if(!(fieldValue instanceof Integer)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }else{
+        return Integer.valueOf(fieldValue.toString());
+      }
+    case DOUBLE:
+      if(!(fieldValue instanceof Double)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }else{
+        return Double.valueOf(fieldValue.toString());
+      }
+    case BIGINT:
+      if(!(fieldValue instanceof Long)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }else{
+        return Long.valueOf(fieldValue.toString());
+      }
+    case FLOAT:
+      if(!(fieldValue instanceof Float)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }else{
+        return Float.valueOf(fieldValue.toString());
+      }
+    case VARCHAR:
+      if(!(fieldValue instanceof String)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }else{
+        return fieldValue.toString();
+      }
+    case TIME:
+    case TIMESTAMP:
+      if(!(fieldValue instanceof Date)){
+        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }else{
+        return fieldValue;
+      }
+    default:
+      throw new UnsupportedDataTypeException(fieldType);
+    }
+  }
+
+  public int size() {
+    return dataValues.size();
+  }
+
+  public List<Object> getDataValues() {
+    return dataValues;
+  }
+
+  public void setDataValues(List<Object> dataValues) {
+    this.dataValues = dataValues;
+  }
+
+  public BeamSQLRecordType getDataType() {
+    return dataType;
+  }
+
+  public void setDataType(BeamSQLRecordType dataType) {
+    this.dataType = dataType;
+  }
+
+  public void setNullFields(List<Integer> nullFields) {
+    this.nullFields = nullFields;
+  }
+
+  public List<Integer> getNullFields() {
+    return nullFields;
+  }
+
+  @Override
+  public String toString() {
+    return "BeamSQLRow [dataValues=" + dataValues + ", dataType=" + dataType + "]";
+  }
+
+  /**
+   * Return data fields as key=value.
+   */
+  public String valueInString() {
+    StringBuffer sb = new StringBuffer();
+    for (int idx = 0; idx < size(); ++idx) {
+      sb.append(String.format(",%s=%s", dataType.getFieldsName().get(idx), getFieldValue(idx)));
+    }
+    return sb.substring(1);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+    BeamSQLRow other = (BeamSQLRow) obj;
+    return toString().equals(other.toString());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSqlRowCoder.java
new file mode 100644
index 000000000000..00af18d64f74
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Date;
+import java.util.List;
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.coders.BigEndianLongCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.DoubleCoder;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.coders.StandardCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+
+/**
+ * A {@link Coder} encodes {@link BeamSQLRow}.
+ *
+ */
+public class BeamSqlRowCoder extends StandardCoder<BeamSQLRow>{
+  private static final BeamSQLRecordTypeCoder recordTypeCoder = BeamSQLRecordTypeCoder.of();
+
+  private static final ListCoder<Integer> listCoder = ListCoder.of(BigEndianIntegerCoder.of());
+
+  private static final StringUtf8Coder stringCoder = StringUtf8Coder.of();
+  private static final BigEndianIntegerCoder intCoder = BigEndianIntegerCoder.of();
+  private static final BigEndianLongCoder longCoder = BigEndianLongCoder.of();
+  private static final DoubleCoder doubleCoder = DoubleCoder.of();
+
+  private static final BeamSqlRowCoder INSTANCE = new BeamSqlRowCoder();
+  private BeamSqlRowCoder(){}
+
+  public static BeamSqlRowCoder of() {
+    return INSTANCE;
+  }
+
+  @Override
+  public void encode(BeamSQLRow value, OutputStream outStream,
+      org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
+    recordTypeCoder.encode(value.getDataType(), outStream, context);
+    listCoder.encode(value.getNullFields(), outStream, context);
+
+    Context nested = context.nested();
+
+    for (int idx = 0; idx < value.size(); ++idx) {
+      if(value.getNullFields().contains(idx)){
+        continue;
+      }
+
+      switch (value.getDataType().getFieldsType().get(idx)) {
+      case INTEGER:
+      case SMALLINT:
+      case TINYINT:
+        intCoder.encode(value.getInteger(idx), outStream, nested);
+        break;
+      case DOUBLE:
+      case FLOAT:
+        doubleCoder.encode(value.getDouble(idx), outStream, nested);
+        break;
+      case BIGINT:
+        longCoder.encode(value.getLong(idx), outStream, nested);
+        break;
+      case VARCHAR:
+        stringCoder.encode(value.getString(idx), outStream, nested);
+        break;
+      case TIME:
+      case TIMESTAMP:
+        longCoder.encode(value.getDate(idx).getTime(), outStream, nested);
+        break;
+
+      default:
+        throw new UnsupportedDataTypeException(value.getDataType().getFieldsType().get(idx));
+      }
+    }
+  }
+
+  @Override
+  public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+      throws CoderException, IOException {
+    BeamSQLRecordType type = recordTypeCoder.decode(inStream, context);
+    List<Integer> nullFields = listCoder.decode(inStream, context);
+
+    BeamSQLRow record = new BeamSQLRow(type);
+    record.setNullFields(nullFields);
+
+    for (int idx = 0; idx < type.size(); ++idx) {
+      if(nullFields.contains(idx)){
+        continue;
+      }
+
+      switch (type.getFieldsType().get(idx)) {
+      case INTEGER:
+      case SMALLINT:
+      case TINYINT:
+        record.addField(idx, intCoder.decode(inStream, context));
+        break;
+      case DOUBLE:
+      case FLOAT:
+        record.addField(idx, doubleCoder.decode(inStream, context));
+        break;
+      case BIGINT:
+        record.addField(idx, longCoder.decode(inStream, context));
+        break;
+      case VARCHAR:
+        record.addField(idx, stringCoder.decode(inStream, context));
+        break;
+      case TIME:
+      case TIMESTAMP:
+        record.addField(idx, new Date(longCoder.decode(inStream, context)));
+        break;
+
+      default:
+        throw new UnsupportedDataTypeException(type.getFieldsType().get(idx));
+      }
+    }
+
+    return record;
+  }
+
+  @Override
+  public List<? extends Coder<?>> getCoderArguments() {
+    return null;
+  }
+
+  @Override
+  public void verifyDeterministic()
+      throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
+
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/InvalidFieldException.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/InvalidFieldException.java
new file mode 100644
index 000000000000..62404269a50c
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/InvalidFieldException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+public class InvalidFieldException extends RuntimeException {
+
+  public InvalidFieldException() {
+    super();
+  }
+
+  public InvalidFieldException(String message) {
+    super(message);
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/UnsupportedDataTypeException.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/UnsupportedDataTypeException.java
new file mode 100644
index 000000000000..9a2235e93c56
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/UnsupportedDataTypeException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema;
+
+import org.apache.calcite.sql.type.SqlTypeName;
+
+public class UnsupportedDataTypeException extends RuntimeException {
+
+  public UnsupportedDataTypeException(SqlTypeName unsupportedType){
+    super(String.format("Not support data type [%s]", unsupportedType));
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
new file mode 100644
index 000000000000..2570763c3e7b
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema.kafka;
+
+import java.util.List;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A Kafka topic that saves records as CSV format.
+ *
+ */
+public class BeamKafkaCSVTable extends BeamKafkaTable {
+
+  /**
+   *
+   */
+  private static final long serialVersionUID = 4754022536543333984L;
+
+  public static final String DELIMITER = ",";
+  private static final Logger LOG = LoggerFactory.getLogger(BeamKafkaCSVTable.class);
+
+  public BeamKafkaCSVTable(RelProtoDataType protoRowType, String bootstrapServers,
+      List<String> topics) {
+    super(protoRowType, bootstrapServers, topics);
+  }
+
+  @Override
+  public PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamSQLRow>>
+      getPTransformForInput() {
+    return new CsvRecorderDecoder(beamSqlRecordType);
+  }
+
+  @Override
+  public PTransform<PCollection<BeamSQLRow>, PCollection<KV<byte[], byte[]>>>
+      getPTransformForOutput() {
+    return new CsvRecorderEncoder(beamSqlRecordType);
+  }
+
+  /**
+   * A PTransform to convert {@code KV<byte[], byte[]>} to {@link BeamSQLRow}.
+   *
+   */
+  public static class CsvRecorderDecoder
+      extends PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamSQLRow>> {
+    private BeamSQLRecordType recordType;
+
+    public CsvRecorderDecoder(BeamSQLRecordType recordType) {
+      this.recordType = recordType;
+    }
+
+    @Override
+    public PCollection<BeamSQLRow> expand(PCollection<KV<byte[], byte[]>> input) {
+      return input.apply("decodeRecord", ParDo.of(new DoFn<KV<byte[], byte[]>, BeamSQLRow>() {
+        @ProcessElement
+        public void processElement(ProcessContext c) {
+          String rowInString = new String(c.element().getValue());
+          String[] parts = rowInString.split(BeamKafkaCSVTable.DELIMITER);
+          if (parts.length != recordType.size()) {
+            LOG.error(String.format("invalid record: ", rowInString));
+          } else {
+            BeamSQLRow sourceRecord = new BeamSQLRow(recordType);
+            for (int idx = 0; idx < parts.length; ++idx) {
+              sourceRecord.addField(idx, parts[idx]);
+            }
+            c.output(sourceRecord);
+          }
+        }
+      }));
+    }
+  }
+
+  /**
+   * A PTransform to convert {@link BeamSQLRow} to {@code KV<byte[], byte[]>}.
+   *
+   */
+  public static class CsvRecorderEncoder
+      extends PTransform<PCollection<BeamSQLRow>, PCollection<KV<byte[], byte[]>>> {
+    private BeamSQLRecordType recordType;
+
+    public CsvRecorderEncoder(BeamSQLRecordType recordType) {
+      this.recordType = recordType;
+    }
+
+    @Override
+    public PCollection<KV<byte[], byte[]>> expand(PCollection<BeamSQLRow> input) {
+      return input.apply("encodeRecord", ParDo.of(new DoFn<BeamSQLRow, KV<byte[], byte[]>>() {
+        @ProcessElement
+        public void processElement(ProcessContext c) {
+          BeamSQLRow in = c.element();
+          StringBuffer sb = new StringBuffer();
+          for (int idx = 0; idx < in.size(); ++idx) {
+            sb.append(DELIMITER);
+            sb.append(in.getFieldValue(idx).toString());
+          }
+          c.output(KV.of(new byte[] {}, sb.substring(1).getBytes()));
+        }
+      }));
+
+    }
+
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaTable.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
new file mode 100644
index 000000000000..482383bc08c7
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.schema.kafka;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.io.kafka.KafkaIO;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.beam.dsls.sql.schema.BeamIOType;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ * {@code BeamKafkaTable} represent a Kafka topic, as source or target. Need to
+ * extend to convert between {@code BeamSQLRow} and {@code KV<byte[], byte[]>}.
+ *
+ */
+public abstract class BeamKafkaTable extends BaseBeamTable implements Serializable {
+
+  /**
+   *
+   */
+  private static final long serialVersionUID = -634715473399906527L;
+
+  private String bootstrapServers;
+  private List<String> topics;
+  private Map<String, Object> configUpdates;
+
+  protected BeamKafkaTable(RelProtoDataType protoRowType) {
+    super(protoRowType);
+  }
+
+  public BeamKafkaTable(RelProtoDataType protoRowType, String bootstrapServers,
+      List<String> topics) {
+    super(protoRowType);
+    this.bootstrapServers = bootstrapServers;
+    this.topics = topics;
+  }
+
+  public BeamKafkaTable updateConsumerProperties(Map<String, Object> configUpdates) {
+    this.configUpdates = configUpdates;
+    return this;
+  }
+
+  @Override
+  public BeamIOType getSourceType() {
+    return BeamIOType.UNBOUNDED;
+  }
+
+  public abstract PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamSQLRow>>
+      getPTransformForInput();
+
+  public abstract PTransform<PCollection<BeamSQLRow>, PCollection<KV<byte[], byte[]>>>
+      getPTransformForOutput();
+
+  @Override
+  public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
+    return new PTransform<PBegin, PCollection<BeamSQLRow>>() {
+
+      @Override
+      public PCollection<BeamSQLRow> expand(PBegin input) {
+        return input.apply("read",
+            KafkaIO.<byte[], byte[]>read().withBootstrapServers(bootstrapServers).withTopics(topics)
+                .updateConsumerProperties(configUpdates).withKeyCoder(ByteArrayCoder.of())
+                .withValueCoder(ByteArrayCoder.of()).withoutMetadata())
+            .apply("in_format", getPTransformForInput());
+
+      }
+    };
+  }
+
+  @Override
+  public PTransform<? super PCollection<BeamSQLRow>, PDone> buildIOWriter() {
+    checkArgument(topics != null && topics.size() == 1,
+        "Only one topic can be acceptable as output.");
+
+    return new PTransform<PCollection<BeamSQLRow>, PDone>() {
+      @Override
+      public PDone expand(PCollection<BeamSQLRow> input) {
+        return input.apply("out_reformat", getPTransformForOutput()).apply("persistent",
+            KafkaIO.<byte[], byte[]>write().withBootstrapServers(bootstrapServers)
+                .withTopic(topics.get(0)).withKeyCoder(ByteArrayCoder.of())
+                .withValueCoder(ByteArrayCoder.of()));
+      }
+    };
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/package-info.java
new file mode 100644
index 000000000000..822fce703da1
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * table schema for KafkaIO.
+ */
+package org.beam.dsls.sql.schema.kafka;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/package-info.java
new file mode 100644
index 000000000000..ef9cc7d38b6a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/schema/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * define table schema, to map with Beam IO components.
+ *
+ */
+package org.beam.dsls.sql.schema;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLFilterFn.java b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLFilterFn.java
new file mode 100644
index 000000000000..06db2802c393
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLFilterFn.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.transform;
+
+import java.util.List;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.beam.dsls.sql.rel.BeamFilterRel;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ * {@code BeamSQLFilterFn} is the executor for a {@link BeamFilterRel} step.
+ *
+ */
+public class BeamSQLFilterFn extends DoFn<BeamSQLRow, BeamSQLRow> {
+  /**
+   *
+   */
+  private static final long serialVersionUID = -1256111753670606705L;
+
+  private String stepName;
+  private BeamSQLExpressionExecutor executor;
+
+  public BeamSQLFilterFn(String stepName, BeamSQLExpressionExecutor executor) {
+    super();
+    this.stepName = stepName;
+    this.executor = executor;
+  }
+
+  @Setup
+  public void setup() {
+    executor.prepare();
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) {
+    BeamSQLRow in = c.element();
+
+    List<Object> result = executor.execute(in);
+
+    if ((Boolean) result.get(0)) {
+      c.output(in);
+    }
+  }
+
+  @Teardown
+  public void close() {
+    executor.close();
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
new file mode 100644
index 000000000000..1014c0d3f7b1
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.transform;
+
+import org.apache.beam.sdk.transforms.DoFn;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ * A test PTransform to display output in console.
+ *
+ */
+public class BeamSQLOutputToConsoleFn extends DoFn<BeamSQLRow, Void> {
+  /**
+   *
+   */
+  private static final long serialVersionUID = -1256111753670606705L;
+
+  private String stepName;
+
+  public BeamSQLOutputToConsoleFn(String stepName) {
+    super();
+    this.stepName = stepName;
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) {
+    System.out.println("Output: " + c.element().getDataValues());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLProjectFn.java b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLProjectFn.java
new file mode 100644
index 000000000000..12061d2f094c
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLProjectFn.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.transform;
+
+import java.util.List;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.beam.dsls.sql.rel.BeamProjectRel;
+import org.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ *
+ * {@code BeamSQLProjectFn} is the executor for a {@link BeamProjectRel} step.
+ *
+ */
+public class BeamSQLProjectFn extends DoFn<BeamSQLRow, BeamSQLRow> {
+
+  /**
+   *
+   */
+  private static final long serialVersionUID = -1046605249999014608L;
+  private String stepName;
+  private BeamSQLExpressionExecutor executor;
+  private BeamSQLRecordType outputRecordType;
+
+  public BeamSQLProjectFn(String stepName, BeamSQLExpressionExecutor executor,
+      BeamSQLRecordType outputRecordType) {
+    super();
+    this.stepName = stepName;
+    this.executor = executor;
+    this.outputRecordType = outputRecordType;
+  }
+
+  @Setup
+  public void setup() {
+    executor.prepare();
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) {
+    List<Object> results = executor.execute(c.element());
+
+    BeamSQLRow outRow = new BeamSQLRow(outputRecordType);
+    for (int idx = 0; idx < results.size(); ++idx) {
+      outRow.addField(idx, results.get(idx));
+    }
+
+    c.output(outRow);
+  }
+
+  @Teardown
+  public void close() {
+    executor.close();
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/package-info.java
new file mode 100644
index 000000000000..2607abf2f123
--- /dev/null
+++ b/dsls/sql/src/main/java/org/beam/dsls/sql/transform/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.apache.beam.sdk.transforms.PTransform} used in a BeamSQL pipeline.
+ */
+package org.beam.dsls.sql.transform;
diff --git a/dsls/sql/src/main/resources/log4j.properties b/dsls/sql/src/main/resources/log4j.properties
new file mode 100644
index 000000000000..709484b4951b
--- /dev/null
+++ b/dsls/sql/src/main/resources/log4j.properties
@@ -0,0 +1,23 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=ERROR,console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
\ No newline at end of file
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BasePlanner.java b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BasePlanner.java
new file mode 100644
index 000000000000..56e45c4732d2
--- /dev/null
+++ b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BasePlanner.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
+import org.junit.BeforeClass;
+
+/**
+ * prepare {@code BeamSqlRunner} for test.
+ *
+ */
+public class BasePlanner {
+  public static BeamSqlRunner runner = new BeamSqlRunner();
+
+  @BeforeClass
+  public static void prepare() {
+    runner.addTable("ORDER_DETAILS", getTable());
+    runner.addTable("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
+    runner.addTable("SUB_ORDER_RAM", getTable());
+  }
+
+  private static BaseBeamTable getTable() {
+    final RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder().add("order_id", SqlTypeName.BIGINT).add("site_id", SqlTypeName.INTEGER)
+            .add("price", SqlTypeName.DOUBLE).add("order_time", SqlTypeName.TIMESTAMP).build();
+      }
+    };
+
+    return new MockedBeamSQLTable(protoRowType);
+  }
+
+  public static BaseBeamTable getTable(String bootstrapServer, String topic) {
+    final RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder().add("order_id", SqlTypeName.BIGINT).add("site_id", SqlTypeName.INTEGER)
+            .add("price", SqlTypeName.DOUBLE).add("order_time", SqlTypeName.TIMESTAMP).build();
+      }
+    };
+
+    Map<String, Object> consumerPara = new HashMap<String, Object>();
+    consumerPara.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
+
+    return new BeamKafkaCSVTable(protoRowType, bootstrapServer, Arrays.asList(topic))
+        .updateConsumerProperties(consumerPara);
+  }
+}
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerExplainTest.java b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerExplainTest.java
new file mode 100644
index 000000000000..a77878fc150e
--- /dev/null
+++ b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerExplainTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests to explain queries.
+ *
+ */
+public class BeamPlannerExplainTest extends BasePlanner {
+
+  @Test
+  public void selectAll() throws Exception {
+    String sql = "SELECT * FROM ORDER_DETAILS";
+    String plan = runner.explainQuery(sql);
+
+    String expectedPlan =
+        "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], order_time=[$3])\n"
+        + "  BeamIOSourceRel(table=[[ORDER_DETAILS]])\n";
+    Assert.assertEquals("explain doesn't match", expectedPlan, plan);
+  }
+
+  @Test
+  public void selectWithFilter() throws Exception {
+    String sql = "SELECT " + " order_id, site_id, price " + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 and price > 20";
+    String plan = runner.explainQuery(sql);
+
+    String expectedPlan = "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2])\n"
+        + "  BeamFilterRel(condition=[AND(=($1, 0), >($2, 20))])\n"
+        + "    BeamIOSourceRel(table=[[ORDER_DETAILS]])\n";
+    Assert.assertEquals("explain doesn't match", expectedPlan, plan);
+  }
+
+  @Test
+  public void insertSelectFilter() throws Exception {
+    String sql = "INSERT INTO SUB_ORDER(order_id, site_id, price) " + "SELECT "
+        + " order_id, site_id, price " + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 and price > 20";
+    String plan = runner.explainQuery(sql);
+
+    String expectedPlan =
+        "BeamIOSinkRel(table=[[SUB_ORDER]], operation=[INSERT], flattened=[true])\n"
+        + "  BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], order_time=[null])\n"
+        + "    BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2])\n"
+        + "      BeamFilterRel(condition=[AND(=($1, 0), >($2, 20))])\n"
+        + "        BeamIOSourceRel(table=[[ORDER_DETAILS]])\n";
+    Assert.assertEquals("explain doesn't match", expectedPlan, plan);
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerSubmitTest.java b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
new file mode 100644
index 000000000000..eb097a9ed293
--- /dev/null
+++ b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import org.apache.beam.sdk.Pipeline;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests to execute a query.
+ *
+ */
+public class BeamPlannerSubmitTest extends BasePlanner {
+  @Test
+  public void insertSelectFilter() throws Exception {
+    String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
+        + " order_id, site_id, price " + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 and price > 20";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    runner.getPlanner().planner.close();
+
+    pipeline.run().waitUntilFinish();
+
+    Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
+    Assert.assertEquals("order_id=12345,site_id=0,price=20.5,order_time=null", MockedBeamSQLTable.CONTENT.get(0));
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/MockedBeamSQLTable.java b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/MockedBeamSQLTable.java
new file mode 100644
index 000000000000..31f55780c395
--- /dev/null
+++ b/dsls/sql/src/test/java/org/beam/dsls/sql/planner/MockedBeamSQLTable.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.beam.dsls.sql.planner;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.beam.dsls.sql.schema.BaseBeamTable;
+import org.beam.dsls.sql.schema.BeamIOType;
+import org.beam.dsls.sql.schema.BeamSQLRow;
+
+/**
+ * A mock table use to check input/output.
+ *
+ */
+public class MockedBeamSQLTable extends BaseBeamTable {
+
+  /**
+   *
+   */
+  private static final long serialVersionUID = 1373168368414036932L;
+
+  public static final List<String> CONTENT = new ArrayList<>();
+
+  public MockedBeamSQLTable(RelProtoDataType protoRowType) {
+    super(protoRowType);
+  }
+
+  @Override
+  public BeamIOType getSourceType() {
+    return BeamIOType.UNBOUNDED;
+  }
+
+  @Override
+  public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
+    BeamSQLRow row1 = new BeamSQLRow(beamSqlRecordType);
+    row1.addField(0, 12345L);
+    row1.addField(1, 0);
+    row1.addField(2, 10.5);
+    row1.addField(3, new Date());
+
+    BeamSQLRow row2 = new BeamSQLRow(beamSqlRecordType);
+    row2.addField(0, 12345L);
+    row2.addField(1, 1);
+    row2.addField(2, 20.5);
+    row2.addField(3, new Date());
+
+    BeamSQLRow row3 = new BeamSQLRow(beamSqlRecordType);
+    row3.addField(0, 12345L);
+    row3.addField(1, 0);
+    row3.addField(2, 20.5);
+    row3.addField(3, new Date());
+
+    BeamSQLRow row4 = new BeamSQLRow(beamSqlRecordType);
+    row4.addField(0, null);
+    row4.addField(1, null);
+    row4.addField(2, 20.5);
+    row4.addField(3, new Date());
+
+    return Create.of(row1, row2, row3);
+  }
+
+  @Override
+  public PTransform<? super PCollection<BeamSQLRow>, PDone> buildIOWriter() {
+    return new OutputStore();
+  }
+
+  /**
+   * Keep output in {@code CONTENT} for validation.
+   *
+   */
+  public static class OutputStore extends PTransform<PCollection<BeamSQLRow>, PDone> {
+
+    @Override
+    public PDone expand(PCollection<BeamSQLRow> input) {
+      input.apply(ParDo.of(new DoFn<BeamSQLRow, Void>() {
+
+        @Setup
+        public void setup() {
+          CONTENT.clear();
+        }
+
+        @ProcessElement
+        public void processElement(ProcessContext c) {
+          CONTENT.add(c.element().valueInString());
+        }
+
+        @Teardown
+        public void close() {
+
+        }
+
+      }));
+      return PDone.in(input.getPipeline());
+    }
+
+  }
+
+}

From 2dcb97397d6c0f7b7b34a9e2ca587b50daec0bdc Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Thu, 13 Apr 2017 14:46:28 -0700
Subject: [PATCH 368/387] checkstyle and rename package

correct package from org.beam.dsls.sql to org.apache.beam.dsls.sql

update with checkstyle
---
 dsls/pom.xml                                  |  7 ++
 .../beam/dsls/sql/example/BeamSqlExample.java |  9 +-
 .../beam/dsls/sql/example/package-info.java   |  2 +-
 .../BeamSQLExpressionExecutor.java            |  4 +-
 .../sql/interpreter/BeamSQLSpELExecutor.java  | 13 +--
 .../dsls/sql/interpreter/CalciteToSpEL.java   |  5 +-
 .../dsls/sql/interpreter/package-info.java    |  2 +-
 .../beam/dsls/sql/package-info.java           |  2 +-
 .../dsls/sql/planner/BeamPipelineCreator.java | 15 ++--
 .../dsls/sql/planner/BeamQueryPlanner.java    | 10 ++-
 .../sql/planner/BeamRelDataTypeSystem.java    |  2 +-
 .../beam/dsls/sql/planner/BeamRuleSets.java   | 13 +--
 .../dsls/sql/planner/BeamSQLRelUtils.java     |  5 +-
 .../beam/dsls/sql/planner/BeamSqlRunner.java  |  7 +-
 .../planner/BeamSqlUnsupportedException.java  |  2 +-
 .../planner/UnsupportedOperatorsVisitor.java  |  2 +-
 .../beam/dsls/sql/planner/package-info.java   |  4 +-
 .../beam/dsls/sql/rel/BeamFilterRel.java      | 14 ++--
 .../beam/dsls/sql/rel/BeamIOSinkRel.java      | 11 +--
 .../beam/dsls/sql/rel/BeamIOSourceRel.java    | 11 +--
 .../dsls/sql/rel/BeamLogicalConvention.java   |  2 +-
 .../beam/dsls/sql/rel/BeamProjectRel.java     | 17 ++--
 .../beam/dsls/sql/rel/BeamRelNode.java        |  4 +-
 .../beam/dsls/sql/rel/package-info.java       |  2 +-
 .../beam/dsls/sql/rule/BeamFilterRule.java    |  6 +-
 .../beam/dsls/sql/rule/BeamIOSinkRule.java    |  7 +-
 .../beam/dsls/sql/rule/BeamIOSourceRule.java  |  6 +-
 .../beam/dsls/sql/rule/BeamProjectRule.java   |  6 +-
 .../beam/dsls/sql/rule/package-info.java      | 23 +++++
 .../beam/dsls/sql/schema/BaseBeamTable.java   |  4 +-
 .../beam/dsls/sql/schema/BeamIOType.java      |  2 +-
 .../dsls/sql/schema/BeamSQLRecordType.java    |  2 +-
 .../sql/schema/BeamSQLRecordTypeCoder.java    | 10 +--
 .../beam/dsls/sql/schema/BeamSQLRow.java      | 84 +++++++++++--------
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java |  6 +-
 .../sql/schema/InvalidFieldException.java     |  6 +-
 .../schema/UnsupportedDataTypeException.java  |  6 +-
 .../sql/schema/kafka/BeamKafkaCSVTable.java   |  6 +-
 .../dsls/sql/schema/kafka/BeamKafkaTable.java |  8 +-
 .../dsls/sql/schema/kafka/package-info.java   |  2 +-
 .../beam/dsls/sql/schema/package-info.java    |  2 +-
 .../dsls/sql/transform/BeamSQLFilterFn.java   |  8 +-
 .../transform/BeamSQLOutputToConsoleFn.java   |  4 +-
 .../dsls/sql/transform/BeamSQLProjectFn.java  | 10 +--
 .../beam/dsls/sql/transform/package-info.java |  2 +-
 .../org/beam/dsls/sql/rule/package-info.java  | 22 -----
 .../beam/dsls/sql/planner/BasePlanner.java    |  6 +-
 .../sql/planner/BeamPlannerExplainTest.java   |  2 +-
 .../sql/planner/BeamPlannerSubmitTest.java    |  5 +-
 .../dsls/sql/planner/MockedBeamSQLTable.java  |  8 +-
 50 files changed, 231 insertions(+), 187 deletions(-)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/example/BeamSqlExample.java (94%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/example/package-info.java (95%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java (92%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java (92%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/interpreter/CalciteToSpEL.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/interpreter/package-info.java (95%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/package-info.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/BeamPipelineCreator.java (86%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/BeamQueryPlanner.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/BeamRelDataTypeSystem.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/BeamRuleSets.java (87%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/BeamSQLRelUtils.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/BeamSqlRunner.java (94%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/BeamSqlUnsupportedException.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java (95%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/planner/package-info.java (87%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rel/BeamFilterRel.java (84%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rel/BeamIOSinkRel.java (91%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rel/BeamIOSourceRel.java (88%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rel/BeamLogicalConvention.java (98%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rel/BeamProjectRel.java (85%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rel/BeamRelNode.java (92%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rel/package-info.java (95%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rule/BeamFilterRule.java (92%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rule/BeamIOSinkRule.java (95%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rule/BeamIOSourceRule.java (91%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/rule/BeamProjectRule.java (92%)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/package-info.java
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/BaseBeamTable.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/BeamIOType.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/BeamSQLRecordType.java (98%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java (92%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/BeamSQLRow.java (67%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/BeamSqlRowCoder.java (97%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/InvalidFieldException.java (91%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/UnsupportedDataTypeException.java (90%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java (96%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/kafka/BeamKafkaTable.java (94%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/kafka/package-info.java (94%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/schema/package-info.java (95%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/transform/BeamSQLFilterFn.java (88%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java (93%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/transform/BeamSQLProjectFn.java (87%)
 rename dsls/sql/src/main/java/org/{ => apache}/beam/dsls/sql/transform/package-info.java (95%)
 delete mode 100644 dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java
 rename dsls/sql/src/test/java/org/{ => apache}/beam/dsls/sql/planner/BasePlanner.java (94%)
 rename dsls/sql/src/test/java/org/{ => apache}/beam/dsls/sql/planner/BeamPlannerExplainTest.java (98%)
 rename dsls/sql/src/test/java/org/{ => apache}/beam/dsls/sql/planner/BeamPlannerSubmitTest.java (93%)
 rename dsls/sql/src/test/java/org/{ => apache}/beam/dsls/sql/planner/MockedBeamSQLTable.java (94%)

diff --git a/dsls/pom.xml b/dsls/pom.xml
index a1bb0ee8b766..6f9d6353596b 100644
--- a/dsls/pom.xml
+++ b/dsls/pom.xml
@@ -52,6 +52,13 @@
         </plugin>
       </plugins>
     </pluginManagement>
+    
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+      </plugin>
+    </plugins>
   </build>
 
 </project>
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/example/BeamSqlExample.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
similarity index 94%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/example/BeamSqlExample.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
index 7fb8def5d76f..d32bc59c74a2 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/example/BeamSqlExample.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
@@ -15,23 +15,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.example;
+package org.apache.beam.dsls.sql.example;
 
 import java.io.Serializable;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.dsls.sql.planner.BeamSqlRunner;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelProtoDataType;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.beam.dsls.sql.planner.BeamSqlRunner;
-import org.beam.dsls.sql.schema.BaseBeamTable;
-import org.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
 
 /**
  * This is one quick example.
+ *
  * <p>Before start, follow https://kafka.apache.org/quickstart to setup a Kafka
  * cluster locally, and run below commands to create required Kafka topics:
  * <pre>
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/example/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/package-info.java
similarity index 95%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/example/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/package-info.java
index ae678e4297fd..52a9fcea9134 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/example/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/package-info.java
@@ -20,4 +20,4 @@
  * examples on how to use BeamSQL.
  *
  */
-package org.beam.dsls.sql.example;
+package org.apache.beam.dsls.sql.example;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
similarity index 92%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
index e9d425dd485a..1285280073b4 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLExpressionExecutor.java
@@ -15,11 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.interpreter;
+package org.apache.beam.dsls.sql.interpreter;
 
 import java.io.Serializable;
 import java.util.List;
-import org.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  * {@code BeamSQLExpressionExecutor} fills the gap between relational
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
similarity index 92%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
index 48306da0dc91..9c9c37f7209a 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
@@ -15,20 +15,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.interpreter;
+package org.apache.beam.dsls.sql.interpreter;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import java.util.ArrayList;
 import java.util.List;
+
+import org.apache.beam.dsls.sql.planner.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.rel.BeamFilterRel;
+import org.apache.beam.dsls.sql.rel.BeamProjectRel;
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
-import org.beam.dsls.sql.planner.BeamSqlUnsupportedException;
-import org.beam.dsls.sql.rel.BeamFilterRel;
-import org.beam.dsls.sql.rel.BeamProjectRel;
-import org.beam.dsls.sql.rel.BeamRelNode;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 import org.springframework.expression.Expression;
 import org.springframework.expression.ExpressionParser;
 import org.springframework.expression.spel.SpelParserConfiguration;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/CalciteToSpEL.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/CalciteToSpEL.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/CalciteToSpEL.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/CalciteToSpEL.java
index c7cbace4e360..6cdc31b6e6ec 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/CalciteToSpEL.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/CalciteToSpEL.java
@@ -15,15 +15,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.interpreter;
+package org.apache.beam.dsls.sql.interpreter;
 
 import com.google.common.base.Joiner;
 import java.util.ArrayList;
 import java.util.List;
+
+import org.apache.beam.dsls.sql.planner.BeamSqlUnsupportedException;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexNode;
-import org.beam.dsls.sql.planner.BeamSqlUnsupportedException;
 
 /**
  * {@code CalciteToSpEL} is used in {@link BeamSQLSpELExecutor}, to convert a
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/package-info.java
similarity index 95%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/package-info.java
index 85235e2dcb0b..178d35f97791 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/interpreter/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/package-info.java
@@ -19,4 +19,4 @@
 /**
  * interpreter generate runnable 'code' to execute SQL relational expressions.
  */
-package org.beam.dsls.sql.interpreter;
+package org.apache.beam.dsls.sql.interpreter;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/package-info.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/package-info.java
index c6f5cf63ba8e..b26e8c4666f5 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/package-info.java
@@ -19,4 +19,4 @@
 /**
  * BeamSQL provides a new interface to run a SQL statement with Beam.
  */
-package org.beam.dsls.sql;
+package org.apache.beam.dsls.sql;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamPipelineCreator.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
similarity index 86%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamPipelineCreator.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
index 5a0c73d2a335..00274a2f2feb 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamPipelineCreator.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
@@ -15,20 +15,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import java.util.Map;
+
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordTypeCoder;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.values.PCollection;
-import org.beam.dsls.sql.rel.BeamRelNode;
-import org.beam.dsls.sql.schema.BaseBeamTable;
-import org.beam.dsls.sql.schema.BeamSQLRecordType;
-import org.beam.dsls.sql.schema.BeamSQLRecordTypeCoder;
-import org.beam.dsls.sql.schema.BeamSQLRow;
-import org.beam.dsls.sql.schema.BeamSqlRowCoder;
 
 /**
  * {@link BeamPipelineCreator} converts a {@link BeamRelNode} tree, into a Beam
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamQueryPlanner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamQueryPlanner.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
index a31ace0a584d..aac86d6b8706 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamQueryPlanner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
@@ -15,13 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.calcite.adapter.java.JavaTypeFactory;
@@ -48,9 +53,6 @@
 import org.apache.calcite.tools.Planner;
 import org.apache.calcite.tools.RelConversionException;
 import org.apache.calcite.tools.ValidationException;
-import org.beam.dsls.sql.rel.BeamLogicalConvention;
-import org.beam.dsls.sql.rel.BeamRelNode;
-import org.beam.dsls.sql.schema.BaseBeamTable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRelDataTypeSystem.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
index bf35296df648..c89a740bc87a 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRelDataTypeSystem.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import org.apache.calcite.rel.type.RelDataTypeSystem;
 import org.apache.calcite.rel.type.RelDataTypeSystemImpl;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRuleSets.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
similarity index 87%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRuleSets.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
index 3f40c271c486..2af31dcbe6f5 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamRuleSets.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
@@ -15,19 +15,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import java.util.Iterator;
+
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.rule.BeamFilterRule;
+import org.apache.beam.dsls.sql.rule.BeamIOSinkRule;
+import org.apache.beam.dsls.sql.rule.BeamIOSourceRule;
+import org.apache.beam.dsls.sql.rule.BeamProjectRule;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.tools.RuleSet;
-import org.beam.dsls.sql.rel.BeamRelNode;
-import org.beam.dsls.sql.rule.BeamFilterRule;
-import org.beam.dsls.sql.rule.BeamIOSinkRule;
-import org.beam.dsls.sql.rule.BeamIOSourceRule;
-import org.beam.dsls.sql.rule.BeamProjectRule;
 
 /**
  * {@link RuleSet} used in {@link BeamQueryPlanner}. It translates a standard
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSQLRelUtils.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSQLRelUtils.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSQLRelUtils.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSQLRelUtils.java
index 94b341c82dba..5e5f2155ce55 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSQLRelUtils.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSQLRelUtils.java
@@ -15,14 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.plan.volcano.RelSubset;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.sql.SqlExplainLevel;
-import org.beam.dsls.sql.rel.BeamRelNode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlRunner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
similarity index 94%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlRunner.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
index 9581fcdec50d..e457e8085fc9 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlRunner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
@@ -15,9 +15,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import java.io.Serializable;
+
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.schema.Schema;
 import org.apache.calcite.schema.SchemaPlus;
@@ -25,8 +28,6 @@
 import org.apache.calcite.tools.Frameworks;
 import org.apache.calcite.tools.RelConversionException;
 import org.apache.calcite.tools.ValidationException;
-import org.beam.dsls.sql.rel.BeamRelNode;
-import org.beam.dsls.sql.schema.BaseBeamTable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlUnsupportedException.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
index a3475bb1c261..7cb524351377 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 /**
  * Generic exception for un-supported operations.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
similarity index 95%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
index 702381df60be..9dfa21d8ca51 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import org.apache.calcite.sql.util.SqlShuttle;
 
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/package-info.java
similarity index 87%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/planner/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/package-info.java
index d98c58424121..0506c5ba9896 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/planner/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/package-info.java
@@ -17,8 +17,8 @@
  */
 
 /**
- * {@link org.beam.dsls.sql.planner.BeamQueryPlanner} is the main interface.
+ * {@link org.apache.beam.dsls.sql.planner.BeamQueryPlanner} is the main interface.
  * It defines data sources, validate a SQL statement, and convert it as a Beam
  * pipeline.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamFilterRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
similarity index 84%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamFilterRel.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
index 64f2d1fa6700..10dd1bec559f 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamFilterRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
@@ -15,8 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rel;
+package org.apache.beam.dsls.sql.rel;
 
+import org.apache.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.transform.BeamSQLFilterFn;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
@@ -25,12 +31,6 @@
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rex.RexNode;
-import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
-import org.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
-import org.beam.dsls.sql.planner.BeamPipelineCreator;
-import org.beam.dsls.sql.planner.BeamSQLRelUtils;
-import org.beam.dsls.sql.schema.BeamSQLRow;
-import org.beam.dsls.sql.transform.BeamSQLFilterFn;
 
 /**
  * BeamRelNode to replace a {@code Filter} node.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSinkRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
similarity index 91%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSinkRel.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
index 46654e5e372a..cad0b3c7445e 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSinkRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
@@ -15,10 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rel;
+package org.apache.beam.dsls.sql.rel;
 
 import com.google.common.base.Joiner;
 import java.util.List;
+
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.plan.RelOptCluster;
@@ -28,10 +33,6 @@
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.TableModify;
 import org.apache.calcite.rex.RexNode;
-import org.beam.dsls.sql.planner.BeamPipelineCreator;
-import org.beam.dsls.sql.planner.BeamSQLRelUtils;
-import org.beam.dsls.sql.schema.BaseBeamTable;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  * BeamRelNode to replace a {@code TableModify} node.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSourceRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
similarity index 88%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSourceRel.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
index f14db922e63e..6b1b6cd8c6fe 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamIOSourceRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
@@ -15,19 +15,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rel;
+package org.apache.beam.dsls.sql.rel;
 
 import com.google.common.base.Joiner;
+
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.core.TableScan;
-import org.beam.dsls.sql.planner.BeamPipelineCreator;
-import org.beam.dsls.sql.planner.BeamSQLRelUtils;
-import org.beam.dsls.sql.schema.BaseBeamTable;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  * BeamRelNode to replace a {@code TableScan} node.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamLogicalConvention.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamLogicalConvention.java
similarity index 98%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamLogicalConvention.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamLogicalConvention.java
index 50fe8e013af1..704a374d4c1e 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamLogicalConvention.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamLogicalConvention.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rel;
+package org.apache.beam.dsls.sql.rel;
 
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.plan.ConventionTraitDef;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamProjectRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
similarity index 85%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamProjectRel.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
index e41d74ecd37c..dd731f8a15a1 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamProjectRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
@@ -15,9 +15,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rel;
+package org.apache.beam.dsls.sql.rel;
 
 import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.transform.BeamSQLProjectFn;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
@@ -30,13 +38,6 @@
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
-import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
-import org.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
-import org.beam.dsls.sql.planner.BeamPipelineCreator;
-import org.beam.dsls.sql.planner.BeamSQLRelUtils;
-import org.beam.dsls.sql.schema.BeamSQLRecordType;
-import org.beam.dsls.sql.schema.BeamSQLRow;
-import org.beam.dsls.sql.transform.BeamSQLProjectFn;
 
 /**
  * BeamRelNode to replace a {@code Project} node.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamRelNode.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
similarity index 92%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamRelNode.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
index 07ffee50237c..e50d71a282d3 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/BeamRelNode.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
@@ -15,11 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rel;
+package org.apache.beam.dsls.sql.rel;
 
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.calcite.rel.RelNode;
-import org.beam.dsls.sql.planner.BeamPipelineCreator;
 
 /**
  * A new method {@link #buildBeamPipeline(BeamPipelineCreator)} is added, it's
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/package-info.java
similarity index 95%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rel/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/package-info.java
index 13dc96285942..77d6204591c8 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rel/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/package-info.java
@@ -20,4 +20,4 @@
  * BeamSQL specified nodes, to replace {@link org.apache.calcite.rel.RelNode}.
  *
  */
-package org.beam.dsls.sql.rel;
+package org.apache.beam.dsls.sql.rel;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamFilterRule.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamFilterRule.java
similarity index 92%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamFilterRule.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamFilterRule.java
index 2ad7c074dbdc..414b6665c923 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamFilterRule.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamFilterRule.java
@@ -15,15 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rule;
+package org.apache.beam.dsls.sql.rule;
 
+import org.apache.beam.dsls.sql.rel.BeamFilterRel;
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.convert.ConverterRule;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.logical.LogicalFilter;
-import org.beam.dsls.sql.rel.BeamFilterRel;
-import org.beam.dsls.sql.rel.BeamLogicalConvention;
 
 /**
  * A {@code ConverterRule} to replace {@link Filter} with {@link BeamFilterRel}.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSinkRule.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamIOSinkRule.java
similarity index 95%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSinkRule.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamIOSinkRule.java
index a44c002f05c0..4cc4ef59f691 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSinkRule.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamIOSinkRule.java
@@ -15,9 +15,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rule;
+package org.apache.beam.dsls.sql.rule;
 
 import java.util.List;
+
+import org.apache.beam.dsls.sql.rel.BeamIOSinkRel;
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
@@ -29,8 +32,6 @@
 import org.apache.calcite.rel.logical.LogicalTableModify;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.schema.Table;
-import org.beam.dsls.sql.rel.BeamIOSinkRel;
-import org.beam.dsls.sql.rel.BeamLogicalConvention;
 
 /**
  * A {@code ConverterRule} to replace {@link TableModify} with
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSourceRule.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamIOSourceRule.java
similarity index 91%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSourceRule.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamIOSourceRule.java
index 9e4778b5ff04..85a69ffcfb92 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamIOSourceRule.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamIOSourceRule.java
@@ -15,15 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rule;
+package org.apache.beam.dsls.sql.rule;
 
+import org.apache.beam.dsls.sql.rel.BeamIOSourceRel;
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.convert.ConverterRule;
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.rel.logical.LogicalTableScan;
-import org.beam.dsls.sql.rel.BeamIOSourceRel;
-import org.beam.dsls.sql.rel.BeamLogicalConvention;
 
 /**
  * A {@code ConverterRule} to replace {@link TableScan} with
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamProjectRule.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamProjectRule.java
similarity index 92%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamProjectRule.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamProjectRule.java
index 117a056fa0ff..6dc3b57ed267 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/BeamProjectRule.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamProjectRule.java
@@ -15,15 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.rule;
+package org.apache.beam.dsls.sql.rule;
 
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
+import org.apache.beam.dsls.sql.rel.BeamProjectRel;
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.convert.ConverterRule;
 import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.logical.LogicalProject;
-import org.beam.dsls.sql.rel.BeamLogicalConvention;
-import org.beam.dsls.sql.rel.BeamProjectRel;
 
 /**
  * A {@code ConverterRule} to replace {@link Project} with
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/package-info.java
new file mode 100644
index 000000000000..5d3264784eef
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * {@link org.apache.calcite.plan.RelOptRule} to generate
+ * {@link org.apache.beam.dsls.sql.rel.BeamRelNode}.
+ */
+package org.apache.beam.dsls.sql.rule;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BaseBeamTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/BaseBeamTable.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
index 3816063ab297..81829e9c273f 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BaseBeamTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
@@ -15,9 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
 import java.io.Serializable;
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -31,7 +32,6 @@
 import org.apache.calcite.schema.Schema.TableType;
 import org.apache.calcite.schema.Statistic;
 import org.apache.calcite.schema.Statistics;
-import org.beam.dsls.sql.planner.BeamQueryPlanner;
 
 /**
  * Each IO in Beam has one table schema, by extending {@link BaseBeamTable}.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamIOType.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamIOType.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamIOType.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamIOType.java
index 5e55b0fc6e48..502e8c158370 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamIOType.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamIOType.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
 import java.io.Serializable;
 
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordType.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
similarity index 98%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordType.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
index dc8e38103cc6..661b155cc85a 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordType.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
 import java.io.Serializable;
 import java.util.ArrayList;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
similarity index 92%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
index 2989cb93f4a4..ec330f12352a 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -48,10 +48,10 @@ public void encode(BeamSQLRecordType value, OutputStream outStream,
       org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
     Context nested = context.nested();
     intCoder.encode(value.size(), outStream, nested);
-    for(String fieldName : value.getFieldsName()){
+    for (String fieldName : value.getFieldsName()) {
       stringCoder.encode(fieldName, outStream, nested);
     }
-    for(SqlTypeName fieldType : value.getFieldsType()){
+    for (SqlTypeName fieldType : value.getFieldsType()) {
       stringCoder.encode(fieldType.name(), outStream, nested);
     }
     outStream.flush();
@@ -63,10 +63,10 @@ public BeamSQLRecordType decode(InputStream inStream,
     BeamSQLRecordType typeRecord = new BeamSQLRecordType();
     Context nested = context.nested();
     int size = intCoder.decode(inStream, nested);
-    for(int idx=0; idx<size; ++idx){
+    for (int idx = 0; idx < size; ++idx) {
       typeRecord.getFieldsName().add(stringCoder.decode(inStream, nested));
     }
-    for(int idx=0; idx<size; ++idx){
+    for (int idx = 0; idx < size; ++idx) {
       typeRecord.getFieldsType().add(SqlTypeName.valueOf(stringCoder.decode(inStream, nested)));
     }
     return typeRecord;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
similarity index 67%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRow.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
index db931688122a..b65e23b5ebf5 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSQLRow.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -40,7 +40,7 @@ public class BeamSQLRow implements Serializable {
   public BeamSQLRow(BeamSQLRecordType dataType) {
     this.dataType = dataType;
     this.dataValues = new ArrayList<>();
-    for(int idx=0; idx<dataType.size(); ++idx){
+    for (int idx = 0; idx < dataType.size(); ++idx) {
       dataValues.add(null);
     }
   }
@@ -55,9 +55,11 @@ public void addField(String fieldName, Object fieldValue) {
   }
 
   public void addField(int index, Object fieldValue) {
-    if(fieldValue == null){
+    if (fieldValue == null) {
       dataValues.set(index, fieldValue);
-      if(!nullFields.contains(index)){nullFields.add(index);}
+      if (!nullFields.contains(index)) {
+        nullFields.add(index);
+      }
       return;
     }
 
@@ -66,34 +68,40 @@ public void addField(int index, Object fieldValue) {
     case INTEGER:
     case SMALLINT:
     case TINYINT:
-      if(!(fieldValue instanceof Integer)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      if (!(fieldValue instanceof Integer)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
     case DOUBLE:
-      if(!(fieldValue instanceof Double)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      if (!(fieldValue instanceof Double)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
     case BIGINT:
-      if(!(fieldValue instanceof Long)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      if (!(fieldValue instanceof Long)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
     case FLOAT:
-      if(!(fieldValue instanceof Float)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      if (!(fieldValue instanceof Float)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
     case VARCHAR:
-      if(!(fieldValue instanceof String)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      if (!(fieldValue instanceof String)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
     case TIME:
     case TIMESTAMP:
-      if(!(fieldValue instanceof Date)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      if (!(fieldValue instanceof Date)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
     default:
@@ -128,7 +136,7 @@ public Object getFieldValue(String fieldName) {
   }
 
   public Object getFieldValue(int fieldIdx) {
-    if(nullFields.contains(fieldIdx)){
+    if (nullFields.contains(fieldIdx)) {
       return null;
     }
 
@@ -139,40 +147,46 @@ public Object getFieldValue(int fieldIdx) {
     case INTEGER:
     case SMALLINT:
     case TINYINT:
-      if(!(fieldValue instanceof Integer)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }else{
+      if (!(fieldValue instanceof Integer)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
         return Integer.valueOf(fieldValue.toString());
       }
     case DOUBLE:
-      if(!(fieldValue instanceof Double)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }else{
+      if (!(fieldValue instanceof Double)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
         return Double.valueOf(fieldValue.toString());
       }
     case BIGINT:
-      if(!(fieldValue instanceof Long)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }else{
+      if (!(fieldValue instanceof Long)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
         return Long.valueOf(fieldValue.toString());
       }
     case FLOAT:
-      if(!(fieldValue instanceof Float)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }else{
+      if (!(fieldValue instanceof Float)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
         return Float.valueOf(fieldValue.toString());
       }
     case VARCHAR:
-      if(!(fieldValue instanceof String)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }else{
+      if (!(fieldValue instanceof String)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
         return fieldValue.toString();
       }
     case TIME:
     case TIMESTAMP:
-      if(!(fieldValue instanceof Date)){
-        throw new InvalidFieldException(String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }else{
+      if (!(fieldValue instanceof Date)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
         return fieldValue;
       }
     default:
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
similarity index 97%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSqlRowCoder.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index 00af18d64f74..1979a004b97e 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -61,7 +61,7 @@ public void encode(BeamSQLRow value, OutputStream outStream,
     Context nested = context.nested();
 
     for (int idx = 0; idx < value.size(); ++idx) {
-      if(value.getNullFields().contains(idx)){
+      if (value.getNullFields().contains(idx)) {
         continue;
       }
 
@@ -102,7 +102,7 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
     record.setNullFields(nullFields);
 
     for (int idx = 0; idx < type.size(); ++idx) {
-      if(nullFields.contains(idx)){
+      if (nullFields.contains(idx)) {
         continue;
       }
 
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/InvalidFieldException.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/InvalidFieldException.java
similarity index 91%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/InvalidFieldException.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/InvalidFieldException.java
index 62404269a50c..c929a83249e3 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/InvalidFieldException.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/InvalidFieldException.java
@@ -15,8 +15,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
+/**
+ * Exception when the field is invalid.
+ *
+ */
 public class InvalidFieldException extends RuntimeException {
 
   public InvalidFieldException() {
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/UnsupportedDataTypeException.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/UnsupportedDataTypeException.java
similarity index 90%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/UnsupportedDataTypeException.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/UnsupportedDataTypeException.java
index 9a2235e93c56..3a496c2381d1 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/UnsupportedDataTypeException.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/UnsupportedDataTypeException.java
@@ -15,10 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
 
 import org.apache.calcite.sql.type.SqlTypeName;
 
+/**
+ * Exception when {@link SqlTypeName} is not supported.
+ *
+ */
 public class UnsupportedDataTypeException extends RuntimeException {
 
   public UnsupportedDataTypeException(SqlTypeName unsupportedType){
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
similarity index 96%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
index 2570763c3e7b..915a5cc15499 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
@@ -15,17 +15,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema.kafka;
+package org.apache.beam.dsls.sql.schema.kafka;
 
 import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.rel.type.RelProtoDataType;
-import org.beam.dsls.sql.schema.BeamSQLRecordType;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
similarity index 94%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
index 482383bc08c7..adf46218623e 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
@@ -15,12 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.schema.kafka;
+package org.apache.beam.dsls.sql.schema.kafka;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamIOType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.io.kafka.KafkaIO;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -29,9 +32,6 @@
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.calcite.rel.type.RelProtoDataType;
-import org.beam.dsls.sql.schema.BaseBeamTable;
-import org.beam.dsls.sql.schema.BeamIOType;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  * {@code BeamKafkaTable} represent a Kafka topic, as source or target. Need to
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/package-info.java
similarity index 94%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/package-info.java
index 822fce703da1..041837288eef 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/kafka/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/package-info.java
@@ -19,4 +19,4 @@
 /**
  * table schema for KafkaIO.
  */
-package org.beam.dsls.sql.schema.kafka;
+package org.apache.beam.dsls.sql.schema.kafka;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/package-info.java
similarity index 95%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/schema/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/package-info.java
index ef9cc7d38b6a..47de06f0b5e6 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/schema/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/package-info.java
@@ -20,4 +20,4 @@
  * define table schema, to map with Beam IO components.
  *
  */
-package org.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.schema;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLFilterFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLFilterFn.java
similarity index 88%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLFilterFn.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLFilterFn.java
index 06db2802c393..55086e2d9619 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLFilterFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLFilterFn.java
@@ -15,13 +15,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.transform;
+package org.apache.beam.dsls.sql.transform;
 
 import java.util.List;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.apache.beam.dsls.sql.rel.BeamFilterRel;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
-import org.beam.dsls.sql.rel.BeamFilterRel;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  * {@code BeamSQLFilterFn} is the executor for a {@link BeamFilterRel} step.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
similarity index 93%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
index 1014c0d3f7b1..92ebff26c25e 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
@@ -15,10 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.transform;
+package org.apache.beam.dsls.sql.transform;
 
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  * A test PTransform to display output in console.
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLProjectFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
similarity index 87%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLProjectFn.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
index 12061d2f094c..bafdd17f97b1 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/BeamSQLProjectFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
@@ -15,14 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.transform;
+package org.apache.beam.dsls.sql.transform;
 
 import java.util.List;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
+import org.apache.beam.dsls.sql.rel.BeamProjectRel;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
-import org.beam.dsls.sql.rel.BeamProjectRel;
-import org.beam.dsls.sql.schema.BeamSQLRecordType;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  *
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/package-info.java
similarity index 95%
rename from dsls/sql/src/main/java/org/beam/dsls/sql/transform/package-info.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/package-info.java
index 2607abf2f123..cd2bdebe0e84 100644
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/transform/package-info.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/package-info.java
@@ -19,4 +19,4 @@
 /**
  * {@link org.apache.beam.sdk.transforms.PTransform} used in a BeamSQL pipeline.
  */
-package org.beam.dsls.sql.transform;
+package org.apache.beam.dsls.sql.transform;
diff --git a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java b/dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java
deleted file mode 100644
index 56ddcf3241b9..000000000000
--- a/dsls/sql/src/main/java/org/beam/dsls/sql/rule/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * {@link org.apache.calcite.plan.RelOptRule} to generate {@link org.beam.dsls.sql.rel.BeamRelNode}.
- */
-package org.beam.dsls.sql.rule;
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BasePlanner.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
similarity index 94%
rename from dsls/sql/src/test/java/org/beam/dsls/sql/planner/BasePlanner.java
rename to dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
index 56e45c4732d2..733b056d4d70 100644
--- a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BasePlanner.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
@@ -15,19 +15,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelProtoDataType;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.beam.dsls.sql.schema.BaseBeamTable;
-import org.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
 import org.junit.BeforeClass;
 
 /**
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerExplainTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
similarity index 98%
rename from dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerExplainTest.java
rename to dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
index a77878fc150e..9dde0f1e5fb7 100644
--- a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerExplainTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import org.junit.Assert;
 import org.junit.Test;
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
similarity index 93%
rename from dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
rename to dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
index eb097a9ed293..d32b19b3133c 100644
--- a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import org.apache.beam.sdk.Pipeline;
 import org.junit.Assert;
@@ -36,7 +36,8 @@ public void insertSelectFilter() throws Exception {
     pipeline.run().waitUntilFinish();
 
     Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
-    Assert.assertEquals("order_id=12345,site_id=0,price=20.5,order_time=null", MockedBeamSQLTable.CONTENT.get(0));
+    Assert.assertEquals("order_id=12345,site_id=0,price=20.5,order_time=null",
+        MockedBeamSQLTable.CONTENT.get(0));
   }
 
 }
diff --git a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/MockedBeamSQLTable.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
similarity index 94%
rename from dsls/sql/src/test/java/org/beam/dsls/sql/planner/MockedBeamSQLTable.java
rename to dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
index 31f55780c395..8631a6e5a479 100644
--- a/dsls/sql/src/test/java/org/beam/dsls/sql/planner/MockedBeamSQLTable.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
@@ -15,12 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.planner;
 
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamIOType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -29,9 +32,6 @@
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.calcite.rel.type.RelProtoDataType;
-import org.beam.dsls.sql.schema.BaseBeamTable;
-import org.beam.dsls.sql.schema.BeamIOType;
-import org.beam.dsls.sql.schema.BeamSQLRow;
 
 /**
  * A mock table use to check input/output.

From d620c5206a68e2fa7baa61dc33ad3b209a052b80 Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Sat, 22 Apr 2017 22:20:25 -0700
Subject: [PATCH 369/387] redesign BeamSqlExpression to execute Calcite SQL
 expression.

Changes:
1. revert BEAM dependency to 0.6.0 to avoid impact of changes in master branch;
2. updates as discussion during review;

refine BeamSqlRowCoder
---
 dsls/sql/pom.xml                              |  47 +++++-
 .../beam/dsls/sql/example/BeamSqlExample.java |   5 -
 .../BeamInvalidOperatorException.java         |  34 +++++
 .../BeamSqlUnsupportedException.java          |   8 +-
 .../InvalidFieldException.java                |   4 +-
 .../beam/dsls/sql/exception/package-info.java |  23 +++
 .../sql/interpreter/BeamSQLFnExecutor.java    | 140 ++++++++++++++++++
 .../sql/interpreter/BeamSQLSpELExecutor.java  | 127 ----------------
 .../dsls/sql/interpreter/CalciteToSpEL.java   |  81 ----------
 .../operator/BeamSqlAndExpression.java        |  60 ++++++++
 .../operator/BeamSqlCompareExpression.java    |  94 ++++++++++++
 .../operator/BeamSqlEqualExpression.java      |  48 ++++++
 .../operator/BeamSqlExpression.java           |  62 ++++++++
 .../operator/BeamSqlInputRefExpression.java   |  46 ++++++
 .../operator/BeamSqlIsNotNullExpression.java  |  51 +++++++
 .../operator/BeamSqlIsNullExpression.java     |  51 +++++++
 .../BeamSqlLargerThanEqualExpression.java     |  49 ++++++
 .../operator/BeamSqlLargerThanExpression.java |  49 ++++++
 .../BeamSqlLessThanEqualExpression.java       |  49 ++++++
 .../operator/BeamSqlLessThanExpression.java   |  49 ++++++
 .../operator/BeamSqlNotEqualExpression.java   |  48 ++++++
 .../operator/BeamSqlOrExpression.java         |  60 ++++++++
 .../operator/BeamSqlPrimitive.java            | 102 +++++++++++++
 .../interpreter/operator/package-info.java    |  22 +++
 .../dsls/sql/planner/BeamQueryPlanner.java    |   8 +-
 .../beam/dsls/sql/planner/BeamSqlRunner.java  |  15 +-
 .../beam/dsls/sql/rel/BeamFilterRel.java      |   4 +-
 .../beam/dsls/sql/rel/BeamProjectRel.java     |   4 +-
 .../beam/dsls/sql/schema/BaseBeamTable.java   |   5 -
 .../dsls/sql/schema/BeamSQLRecordType.java    |   4 -
 .../sql/schema/BeamSQLRecordTypeCoder.java    |  15 +-
 .../beam/dsls/sql/schema/BeamSQLRow.java      |  76 ++++++----
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java |  48 +++---
 .../sql/schema/kafka/BeamKafkaCSVTable.java   |   5 -
 .../dsls/sql/schema/kafka/BeamKafkaTable.java |   5 -
 .../dsls/sql/transform/BeamSQLFilterFn.java   |   4 -
 .../transform/BeamSQLOutputToConsoleFn.java   |   4 -
 .../dsls/sql/transform/BeamSQLProjectFn.java  |   5 -
 .../interpreter/BeamSQLFnExecutorTest.java    | 101 +++++++++++++
 .../BeamSQLFnExecutorTestBase.java            |  91 ++++++++++++
 .../operator/BeamNullExperssionTest.java      |  53 +++++++
 .../operator/BeamSqlAndOrExpressionTest.java  |  59 ++++++++
 .../BeamSqlCompareExpressionTest.java         | 108 ++++++++++++++
 .../BeamSqlInputRefExpressionTest.java        |  58 ++++++++
 .../operator/BeamSqlPrimitiveTest.java        |  60 ++++++++
 .../beam/dsls/sql/planner/BasePlanner.java    |  28 +++-
 .../sql/planner/BeamPlannerExplainTest.java   |   5 +-
 .../sql/planner/BeamPlannerSubmitTest.java    |   7 +-
 .../dsls/sql/planner/MockedBeamSQLTable.java  |  14 +-
 49 files changed, 1739 insertions(+), 356 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/BeamInvalidOperatorException.java
 rename dsls/sql/src/main/java/org/apache/beam/dsls/sql/{planner => exception}/BeamSqlUnsupportedException.java (85%)
 rename dsls/sql/src/main/java/org/apache/beam/dsls/sql/{schema => exception}/InvalidFieldException.java (90%)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/package-info.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
 delete mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
 delete mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/CalciteToSpEL.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlEqualExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNotNullExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNullExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanEqualExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanEqualExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlNotEqualExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlOrExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/package-info.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTestBase.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamNullExperssionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndOrExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitiveTest.java

diff --git a/dsls/sql/pom.xml b/dsls/sql/pom.xml
index 21c8def16074..e2f09beb1ac1 100644
--- a/dsls/sql/pom.xml
+++ b/dsls/sql/pom.xml
@@ -116,7 +116,42 @@
       </plugin>
     </plugins>
   </build>
-  
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>org.apache.beam</groupId>
+        <artifactId>beam-sdks-java-core</artifactId>
+        <version>0.6.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.beam</groupId>
+        <artifactId>beam-runners-direct-java</artifactId>
+        <version>0.6.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.beam</groupId>
+        <artifactId>beam-sdks-java-io-kafka</artifactId>
+        <version>0.6.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.beam</groupId>
+        <artifactId>beam-runners-core-java</artifactId>
+        <version>0.6.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.beam</groupId>
+        <artifactId>beam-sdks-common-runner-api</artifactId>
+        <version>0.6.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.beam</groupId>
+        <artifactId>beam-runners-core-construction-java</artifactId>
+        <version>0.6.0</version>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
   <dependencies>
     <dependency>
       <groupId>junit</groupId>
@@ -130,6 +165,12 @@
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-sdks-java-core</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-lite</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.beam</groupId>
@@ -141,10 +182,6 @@
       <artifactId>beam-sdks-java-io-kafka</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.springframework</groupId>
-      <artifactId>spring-expression</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
index d32bc59c74a2..303835fd6d0a 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
@@ -66,11 +66,6 @@
  */
 public class BeamSqlExample implements Serializable {
 
-  /**
-   *
-   */
-  private static final long serialVersionUID = 3673487843555563904L;
-
   public static void main(String[] args) throws Exception {
     BeamSqlRunner runner = new BeamSqlRunner();
     runner.addTable("ORDER_DETAILS", getTable("127.0.0.1:9092", "orders"));
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/BeamInvalidOperatorException.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/BeamInvalidOperatorException.java
new file mode 100644
index 000000000000..281ef89c589e
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/BeamInvalidOperatorException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.exception;
+
+/**
+ * operation is not supported.
+ *
+ */
+public class BeamInvalidOperatorException extends RuntimeException {
+
+  public BeamInvalidOperatorException(String string) {
+    super(string);
+  }
+
+  public BeamInvalidOperatorException() {
+    super();
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlUnsupportedException.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/BeamSqlUnsupportedException.java
similarity index 85%
rename from dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/BeamSqlUnsupportedException.java
index 7cb524351377..02e843b59050 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlUnsupportedException.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/BeamSqlUnsupportedException.java
@@ -15,17 +15,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.dsls.sql.planner;
+package org.apache.beam.dsls.sql.exception;
 
 /**
- * Generic exception for un-supported operations.
+ * Generic exception for un-supported features/functions in BeamSQL.
  *
  */
 public class BeamSqlUnsupportedException extends RuntimeException {
-  /**
-   *
-   */
-  private static final long serialVersionUID = 3445015747629217342L;
 
   public BeamSqlUnsupportedException(String string) {
     super(string);
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/InvalidFieldException.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/InvalidFieldException.java
similarity index 90%
rename from dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/InvalidFieldException.java
rename to dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/InvalidFieldException.java
index c929a83249e3..82ebabe363d5 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/InvalidFieldException.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/InvalidFieldException.java
@@ -15,10 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.dsls.sql.schema;
+package org.apache.beam.dsls.sql.exception;
 
 /**
- * Exception when the field is invalid.
+ * Exception when the field value and field type is not compatible.
  *
  */
 public class InvalidFieldException extends RuntimeException {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/package-info.java
new file mode 100644
index 000000000000..619100cef1a3
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/exception/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Exceptions in BeamSQL.
+ *
+ */
+package org.apache.beam.dsls.sql.exception;
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
new file mode 100644
index 000000000000..32e2ffcb50d4
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlAndExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlEqualExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlInputRefExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlIsNotNullExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlIsNullExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlLargerThanEqualExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlLargerThanExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlLessThanEqualExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlLessThanExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlNotEqualExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlOrExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.rel.BeamFilterRel;
+import org.apache.beam.dsls.sql.rel.BeamProjectRel;
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+
+/**
+ * Executor based on {@link BeamSqlExpression} and {@link BeamSqlPrimitive}.
+ * {@code BeamSQLFnExecutor} converts a {@link BeamRelNode} to a {@link BeamSqlExpression},
+ * which can be evaluated against the {@link BeamSQLRow}.
+ *
+ */
+public class BeamSQLFnExecutor implements BeamSQLExpressionExecutor {
+  protected List<BeamSqlExpression> exps;
+
+  public BeamSQLFnExecutor(BeamRelNode relNode) {
+    this.exps = new ArrayList<>();
+    if (relNode instanceof BeamFilterRel) {
+      BeamFilterRel filterNode = (BeamFilterRel) relNode;
+      RexNode condition = filterNode.getCondition();
+      exps.add(buildExpression(condition));
+    } else if (relNode instanceof BeamProjectRel) {
+      BeamProjectRel projectNode = (BeamProjectRel) relNode;
+      List<RexNode> projects = projectNode.getProjects();
+      for (RexNode rexNode : projects) {
+        exps.add(buildExpression(rexNode));
+      }
+    } else {
+      throw new BeamSqlUnsupportedException(
+          String.format("%s is not supported yet", relNode.getClass().toString()));
+    }
+  }
+
+  /**
+   * {@link #buildExpression(RexNode)} visits the operands of {@link RexNode} recursively,
+   * and represent each {@link SqlOperator} with a corresponding {@link BeamSqlExpression}.
+   */
+  private BeamSqlExpression buildExpression(RexNode rexNode) {
+    if (rexNode instanceof RexLiteral) {
+      RexLiteral node = (RexLiteral) rexNode;
+      return BeamSqlPrimitive.of(node.getTypeName(), node.getValue());
+    } else if (rexNode instanceof RexInputRef) {
+      RexInputRef node = (RexInputRef) rexNode;
+      return new BeamSqlInputRefExpression(node.getType().getSqlTypeName(), node.getIndex());
+    } else if (rexNode instanceof RexCall) {
+      RexCall node = (RexCall) rexNode;
+      String opName = node.op.getName();
+      List<BeamSqlExpression> subExps = new ArrayList<>();
+      for (RexNode subNode : node.operands) {
+        subExps.add(buildExpression(subNode));
+      }
+      switch (opName) {
+        case "AND":
+        return new BeamSqlAndExpression(subExps);
+        case "OR":
+          return new BeamSqlOrExpression(subExps);
+
+        case "=":
+          return new BeamSqlEqualExpression(subExps);
+        case "<>=":
+          return new BeamSqlNotEqualExpression(subExps);
+        case ">":
+          return new BeamSqlLargerThanExpression(subExps);
+        case ">=":
+          return new BeamSqlLargerThanEqualExpression(subExps);
+        case "<":
+          return new BeamSqlLessThanExpression(subExps);
+        case "<=":
+          return new BeamSqlLessThanEqualExpression(subExps);
+
+        case "IS NULL":
+          return new BeamSqlIsNullExpression(subExps.get(0));
+        case "IS NOT NULL":
+          return new BeamSqlIsNotNullExpression(subExps.get(0));
+      default:
+        throw new BeamSqlUnsupportedException();
+      }
+    } else {
+      throw new BeamSqlUnsupportedException(
+          String.format("%s is not supported yet", rexNode.getClass().toString()));
+    }
+  }
+
+  @Override
+  public void prepare() {
+  }
+
+  @Override
+  public List<Object> execute(BeamSQLRow inputRecord) {
+    List<Object> results = new ArrayList<>();
+    for (BeamSqlExpression exp : exps) {
+      results.add(exp.evaluate(inputRecord).getValue());
+    }
+    return results;
+  }
+
+  @Override
+  public void close() {
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
deleted file mode 100644
index 9c9c37f7209a..000000000000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLSpELExecutor.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.interpreter;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.beam.dsls.sql.planner.BeamSqlUnsupportedException;
-import org.apache.beam.dsls.sql.rel.BeamFilterRel;
-import org.apache.beam.dsls.sql.rel.BeamProjectRel;
-import org.apache.beam.dsls.sql.rel.BeamRelNode;
-import org.apache.beam.dsls.sql.schema.BeamSQLRow;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
-import org.springframework.expression.Expression;
-import org.springframework.expression.ExpressionParser;
-import org.springframework.expression.spel.SpelParserConfiguration;
-import org.springframework.expression.spel.standard.SpelExpressionParser;
-import org.springframework.expression.spel.support.StandardEvaluationContext;
-
-/**
- * {@code BeamSQLSpELExecutor} is one implementation, to convert Calcite SQL
- * relational expression to SpEL expression.
- *
- */
-public class BeamSQLSpELExecutor implements BeamSQLExpressionExecutor {
-  /**
-   *
-   */
-  private static final long serialVersionUID = 6777232573390074408L;
-
-  private List<String> spelString;
-  private List<Expression> spelExpressions;
-
-  public BeamSQLSpELExecutor(BeamRelNode relNode) {
-    this.spelString = new ArrayList<>();
-    if (relNode instanceof BeamFilterRel) {
-      String filterSpEL = CalciteToSpEL
-          .rexcall2SpEL((RexCall) ((BeamFilterRel) relNode).getCondition());
-      spelString.add(filterSpEL);
-    } else if (relNode instanceof BeamProjectRel) {
-      spelString.addAll(createProjectExps((BeamProjectRel) relNode));
-      // List<ProjectRule> projectRules =
-      // for (int idx = 0; idx < projectRules.size(); ++idx) {
-      // spelString.add(projectRules.get(idx).getProjectExp());
-      // }
-    } else {
-      throw new BeamSqlUnsupportedException(
-          String.format("%s is not supported yet", relNode.getClass().toString()));
-    }
-  }
-
-  @Override
-  public void prepare() {
-    this.spelExpressions = new ArrayList<>();
-
-    SpelParserConfiguration config = new SpelParserConfiguration(true, true);
-    ExpressionParser parser = new SpelExpressionParser(config);
-    for (String el : spelString) {
-      spelExpressions.add(parser.parseExpression(el));
-    }
-  }
-
-  @Override
-  public List<Object> execute(BeamSQLRow inputRecord) {
-    StandardEvaluationContext inContext = new StandardEvaluationContext();
-    inContext.setVariable("in", inputRecord);
-
-    List<Object> results = new ArrayList<>();
-    for (Expression ep : spelExpressions) {
-      results.add(ep.getValue(inContext));
-    }
-    return results;
-  }
-
-  @Override
-  public void close() {
-
-  }
-
-  private List<String> createProjectExps(BeamProjectRel projectRel) {
-    List<String> rules = new ArrayList<>();
-
-    List<RexNode> exps = projectRel.getProjects();
-
-    for (int idx = 0; idx < exps.size(); ++idx) {
-      RexNode node = exps.get(idx);
-      if (node == null) {
-        rules.add("null");
-      }
-
-      if (node instanceof RexLiteral) {
-        rules.add(((RexLiteral) node).getValue() + "");
-      } else {
-        if (node instanceof RexInputRef) {
-          rules.add("#in.getFieldValue(" + ((RexInputRef) node).getIndex() + ")");
-        }
-        if (node instanceof RexCall) {
-          rules.add(CalciteToSpEL.rexcall2SpEL((RexCall) node));
-        }
-      }
-    }
-
-    checkArgument(rules.size() == exps.size(), "missing projects rules after conversion.");
-
-    return rules;
-  }
-
-}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/CalciteToSpEL.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/CalciteToSpEL.java
deleted file mode 100644
index 6cdc31b6e6ec..000000000000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/CalciteToSpEL.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.interpreter;
-
-import com.google.common.base.Joiner;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.beam.dsls.sql.planner.BeamSqlUnsupportedException;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-/**
- * {@code CalciteToSpEL} is used in {@link BeamSQLSpELExecutor}, to convert a
- * relational expression {@link RexCall} to SpEL expression.
- *
- */
-public class CalciteToSpEL {
-
-  public static String rexcall2SpEL(RexCall cdn) {
-    List<String> parts = new ArrayList<>();
-    for (RexNode subcdn : cdn.operands) {
-      if (subcdn instanceof RexCall) {
-        parts.add(rexcall2SpEL((RexCall) subcdn));
-      } else {
-        parts.add(subcdn instanceof RexInputRef
-            ? "#in.getFieldValue(" + ((RexInputRef) subcdn).getIndex() + ")" : subcdn.toString());
-      }
-    }
-
-    String opName = cdn.op.getName();
-    switch (cdn.op.getClass().getSimpleName()) {
-    case "SqlMonotonicBinaryOperator": // +-*
-    case "SqlBinaryOperator": // > < = >= <= <> OR AND || / .
-      switch (cdn.op.getName().toUpperCase()) {
-      case "AND":
-        return String.format(" ( %s ) ", Joiner.on("&&").join(parts));
-      case "OR":
-        return String.format(" ( %s ) ", Joiner.on("||").join(parts));
-      case "=":
-        return String.format(" ( %s ) ", Joiner.on("==").join(parts));
-      case "<>":
-        return String.format(" ( %s ) ", Joiner.on("!=").join(parts));
-      default:
-        return String.format(" ( %s ) ", Joiner.on(cdn.op.getName().toUpperCase()).join(parts));
-      }
-    case "SqlCaseOperator": // CASE
-      return String.format(" (%s ? %s : %s)", parts.get(0), parts.get(1), parts.get(2));
-    case "SqlCastFunction": // CAST
-      return parts.get(0);
-    case "SqlPostfixOperator":
-      switch (opName.toUpperCase()) {
-      case "IS NULL":
-        return String.format(" null == %s ", parts.get(0));
-      case "IS NOT NULL":
-        return String.format(" null != %s ", parts.get(0));
-      default:
-        throw new BeamSqlUnsupportedException();
-      }
-    default:
-      throw new BeamSqlUnsupportedException();
-    }
-  }
-
-}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndExpression.java
new file mode 100644
index 000000000000..55473b529751
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndExpression.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'AND' operation.
+ */
+public class BeamSqlAndExpression extends BeamSqlExpression {
+
+  private BeamSqlAndExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+  public BeamSqlAndExpression(List<BeamSqlExpression> operands) {
+    this(operands, SqlTypeName.BOOLEAN);
+  }
+
+  @Override
+  public boolean accept() {
+    for (BeamSqlExpression exp : operands) {
+      // only accept BOOLEAN expression as operand
+      if (!exp.outputType.equals(SqlTypeName.BOOLEAN)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public BeamSqlPrimitive<Boolean> evaluate(BeamSQLRow inputRecord) {
+    boolean result = true;
+    for (BeamSqlExpression exp : operands) {
+      BeamSqlPrimitive<Boolean> expOut = exp.evaluate(inputRecord);
+      result = result && expOut.getValue();
+      if (!result) {
+        break;
+      }
+    }
+    return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, result);
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpression.java
new file mode 100644
index 000000000000..bfb798d77b8d
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpression.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@link BeamSqlCompareExpression} is used for compare operations.
+ *
+ * <p>See {@link BeamSqlEqualExpression}, {@link BeamSqlLessThanExpression},
+ * {@link BeamSqlLessThanEqualExpression}, {@link BeamSqlLargerThanExpression},
+ * {@link BeamSqlLargerThanEqualExpression} and {@link BeamSqlNotEqualExpression} for more details.
+ *
+ */
+public abstract class BeamSqlCompareExpression extends BeamSqlExpression {
+
+  private BeamSqlCompareExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  public BeamSqlCompareExpression(List<BeamSqlExpression> operands) {
+    this(operands, SqlTypeName.BOOLEAN);
+  }
+
+  /**
+   * Compare operation must have 2 operands.
+   */
+  @Override
+  public boolean accept() {
+    return operands.size() == 2;
+  }
+
+  @Override
+  public BeamSqlPrimitive<Boolean> evaluate(BeamSQLRow inputRecord) {
+    Object leftValue = operands.get(0).evaluate(inputRecord).getValue();
+    Object rightValue = operands.get(1).evaluate(inputRecord).getValue();
+    switch (operands.get(0).outputType) {
+    case BIGINT:
+    case DECIMAL:
+    case DOUBLE:
+    case FLOAT:
+    case INTEGER:
+    case SMALLINT:
+    case TINYINT:
+      return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN,
+          compare((Number) leftValue, (Number) rightValue));
+    case BOOLEAN:
+      return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN,
+          compare((Boolean) leftValue, (Boolean) rightValue));
+    case VARCHAR:
+      return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN,
+          compare((CharSequence) leftValue, (CharSequence) rightValue));
+    default:
+      throw new BeamSqlUnsupportedException(toString());
+    }
+  }
+
+  /**
+   * Compare between String values, mapping to {@link SqlTypeName#VARCHAR}.
+   */
+  public abstract Boolean compare(CharSequence leftValue, CharSequence rightValue);
+
+  /**
+   * Compare between Boolean values, mapping to {@link SqlTypeName#BOOLEAN}.
+   */
+  public abstract Boolean compare(Boolean leftValue, Boolean rightValue);
+
+  /**
+   * Compare between Number values, including {@link SqlTypeName#BIGINT},
+   * {@link SqlTypeName#DECIMAL}, {@link SqlTypeName#DOUBLE}, {@link SqlTypeName#FLOAT},
+   * {@link SqlTypeName#INTEGER}, {@link SqlTypeName#SMALLINT} and {@link SqlTypeName#TINYINT}.
+   */
+  public abstract Boolean compare(Number leftValue, Number rightValue);
+
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlEqualExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlEqualExpression.java
new file mode 100644
index 000000000000..4bc487b6d418
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlEqualExpression.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+
+/**
+ * {@code BeamSqlExpression} for {@code =} operation.
+ */
+public class BeamSqlEqualExpression extends BeamSqlCompareExpression {
+
+  public BeamSqlEqualExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override
+  public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+    return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) == 0;
+  }
+
+  @Override
+  public Boolean compare(Boolean leftValue, Boolean rightValue) {
+    return !(leftValue ^ rightValue);
+  }
+
+  @Override
+  public Boolean compare(Number leftValue, Number rightValue) {
+    return (leftValue == null && rightValue == null)
+        || (leftValue != null && rightValue != null
+              && leftValue.floatValue() == (rightValue).floatValue());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
new file mode 100644
index 000000000000..c44795f5b692
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} is an equivalent expression in BeamSQL, of {@link RexNode} in Calcite.
+ *
+ * <p>An implementation of {@link BeamSqlExpression} takes one or more {@code BeamSqlExpression}
+ * as its operands, and return a value with type {@link SqlTypeName}.
+ *
+ */
+public abstract class BeamSqlExpression implements Serializable{
+  protected List<BeamSqlExpression> operands;
+  protected SqlTypeName outputType;
+
+  protected BeamSqlExpression(){}
+
+  public BeamSqlExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    this.operands = operands;
+    this.outputType = outputType;
+  }
+
+  /**
+   * assertion to make sure the input and output are supported in this expression.
+   */
+  public abstract boolean accept();
+
+  /**
+   * Apply input record {@link BeamSQLRow} to this expression,
+   * the output value is wrapped with {@link BeamSqlPrimitive}.
+   */
+  public abstract BeamSqlPrimitive evaluate(BeamSQLRow inputRecord);
+
+  public List<BeamSqlExpression> getOperands() {
+    return operands;
+  }
+
+  public SqlTypeName getOutputType() {
+    return outputType;
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java
new file mode 100644
index 000000000000..612108ff54fb
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * An primitive operation for direct field extraction.
+ */
+public class BeamSqlInputRefExpression extends BeamSqlExpression{
+  private int inputRef;
+
+  public BeamSqlInputRefExpression(SqlTypeName sqlTypeName, int inputRef) {
+    super(null, sqlTypeName);
+    this.inputRef = inputRef;
+  }
+
+  @Override
+  public boolean accept() {
+    // TODO Auto-generated method stub
+    return false;
+  }
+
+  @Override
+  public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    return BeamSqlPrimitive.of(outputType, inputRecord.getFieldValue(inputRef));
+  }
+
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNotNullExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNotNullExpression.java
new file mode 100644
index 000000000000..784584e5e80f
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNotNullExpression.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'IS NOT NULL' operation.
+ */
+public class BeamSqlIsNotNullExpression extends BeamSqlExpression {
+
+  private BeamSqlIsNotNullExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  public BeamSqlIsNotNullExpression(BeamSqlExpression operand){
+    this(Arrays.asList(operand), SqlTypeName.BOOLEAN);
+  }
+
+  /**
+   * only one operand is required.
+   */
+  @Override
+  public boolean accept() {
+    return operands.size() == 1;
+  }
+
+  @Override
+  public BeamSqlPrimitive<Boolean> evaluate(BeamSQLRow inputRecord) {
+    Object leftValue = operands.get(0).evaluate(inputRecord).getValue();
+    return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, leftValue != null);
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNullExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNullExpression.java
new file mode 100644
index 000000000000..b09ddbf3112b
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlIsNullExpression.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'IS NULL' operation.
+ */
+public class BeamSqlIsNullExpression extends BeamSqlExpression {
+
+  private BeamSqlIsNullExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  public BeamSqlIsNullExpression(BeamSqlExpression operand){
+    this(Arrays.asList(operand), SqlTypeName.BOOLEAN);
+  }
+
+  /**
+   * only one operand is required.
+   */
+  @Override
+  public boolean accept() {
+    return operands.size() == 1;
+  }
+
+  @Override
+  public BeamSqlPrimitive<Boolean> evaluate(BeamSQLRow inputRecord) {
+    Object leftValue = operands.get(0).evaluate(inputRecord).getValue();
+    return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, leftValue == null);
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanEqualExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanEqualExpression.java
new file mode 100644
index 000000000000..d78c020169be
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanEqualExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+
+/**
+ * {@code BeamSqlExpression} for {@code >=} operation.
+ */
+public class BeamSqlLargerThanEqualExpression extends BeamSqlCompareExpression {
+
+  public BeamSqlLargerThanEqualExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override
+  public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+    return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) >= 0;
+  }
+
+  @Override
+  public Boolean compare(Boolean leftValue, Boolean rightValue) {
+    throw new BeamInvalidOperatorException(">= is not supported for Boolean.");
+  }
+
+  @Override
+  public Boolean compare(Number leftValue, Number rightValue) {
+    return (leftValue == null && rightValue == null)
+        || (leftValue != null && rightValue != null
+              && leftValue.floatValue() >= (rightValue).floatValue());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanExpression.java
new file mode 100644
index 000000000000..0b0d6f17dee3
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLargerThanExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+
+/**
+ * {@code BeamSqlExpression} for {@code >} operation.
+ */
+public class BeamSqlLargerThanExpression extends BeamSqlCompareExpression {
+
+  public BeamSqlLargerThanExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override
+  public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+    return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) > 0;
+  }
+
+  @Override
+  public Boolean compare(Boolean leftValue, Boolean rightValue) {
+    throw new BeamInvalidOperatorException("> is not supported for Boolean.");
+  }
+
+  @Override
+  public Boolean compare(Number leftValue, Number rightValue) {
+    return (leftValue == null && rightValue == null)
+        || (leftValue != null && rightValue != null
+              && leftValue.floatValue() > (rightValue).floatValue());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanEqualExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanEqualExpression.java
new file mode 100644
index 000000000000..b6f7c9aa0f9f
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanEqualExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+
+/**
+ * {@code BeamSqlExpression} for {@code <=} operation.
+ */
+public class BeamSqlLessThanEqualExpression extends BeamSqlCompareExpression {
+
+  public BeamSqlLessThanEqualExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override
+  public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+    return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) <= 0;
+  }
+
+  @Override
+  public Boolean compare(Boolean leftValue, Boolean rightValue) {
+    throw new BeamInvalidOperatorException("<= is not supported for Boolean.");
+  }
+
+  @Override
+  public Boolean compare(Number leftValue, Number rightValue) {
+    return (leftValue == null && rightValue == null)
+        || (leftValue != null && rightValue != null
+              && leftValue.floatValue() <= (rightValue).floatValue());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanExpression.java
new file mode 100644
index 000000000000..216a621f5d4a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlLessThanExpression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+
+/**
+ * {@code BeamSqlExpression} for {@code <} operation.
+ */
+public class BeamSqlLessThanExpression extends BeamSqlCompareExpression {
+
+  public BeamSqlLessThanExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override
+  public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+    return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) < 0;
+  }
+
+  @Override
+  public Boolean compare(Boolean leftValue, Boolean rightValue) {
+    throw new BeamInvalidOperatorException("< is not supported for Boolean.");
+  }
+
+  @Override
+  public Boolean compare(Number leftValue, Number rightValue) {
+    return (leftValue == null && rightValue == null)
+        || (leftValue != null && rightValue != null
+              && leftValue.floatValue() < (rightValue).floatValue());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlNotEqualExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlNotEqualExpression.java
new file mode 100644
index 000000000000..2b093bfa7595
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlNotEqualExpression.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+
+/**
+ * {@code BeamSqlExpression} for {@code <>} operation.
+ */
+public class BeamSqlNotEqualExpression extends BeamSqlCompareExpression {
+
+  public BeamSqlNotEqualExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override
+  public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
+    return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) != 0;
+  }
+
+  @Override
+  public Boolean compare(Boolean leftValue, Boolean rightValue) {
+    return leftValue ^ rightValue;
+  }
+
+  @Override
+  public Boolean compare(Number leftValue, Number rightValue) {
+    return (leftValue == null && rightValue == null)
+        || (leftValue != null && rightValue != null
+              && leftValue.floatValue() != (rightValue).floatValue());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlOrExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlOrExpression.java
new file mode 100644
index 000000000000..4d07af8f8d8f
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlOrExpression.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for 'OR' operation.
+ */
+public class BeamSqlOrExpression extends BeamSqlExpression {
+
+  private BeamSqlOrExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+  public BeamSqlOrExpression(List<BeamSqlExpression> operands) {
+    this(operands, SqlTypeName.BOOLEAN);
+  }
+
+  @Override
+  public boolean accept() {
+    for (BeamSqlExpression exp : operands) {
+      // only accept BOOLEAN expression as operand
+      if (!exp.outputType.equals(SqlTypeName.BOOLEAN)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public BeamSqlPrimitive<Boolean> evaluate(BeamSQLRow inputRecord) {
+    boolean result = false;
+    for (BeamSqlExpression exp : operands) {
+      BeamSqlPrimitive<Boolean> expOut = exp.evaluate(inputRecord);
+        result = result || expOut.getValue();
+        if (result) {
+          break;
+        }
+    }
+    return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, result);
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
new file mode 100644
index 000000000000..71852ffcd235
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.math.BigDecimal;
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@link BeamSqlPrimitive} is a special, self-reference {@link BeamSqlExpression}.
+ * It holds the value, and return it directly during {@link #evaluate(BeamSQLRow)}.
+ *
+ */
+public class BeamSqlPrimitive<T> extends BeamSqlExpression{
+  private SqlTypeName outputType;
+  private T value;
+
+  private BeamSqlPrimitive() {
+  }
+
+  private BeamSqlPrimitive(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  /**
+   * A builder function to create from Type and value directly.
+   */
+  public static <T> BeamSqlPrimitive<T> of(SqlTypeName outputType, T value){
+    BeamSqlPrimitive<T> exp = new BeamSqlPrimitive<T>();
+    exp.outputType = outputType;
+    exp.value = value;
+    if (!exp.accept()) {
+      throw new BeamInvalidOperatorException(
+          String.format("value [%s] doesn't match type [%s].", value, outputType));
+    }
+    return exp;
+  }
+
+  public SqlTypeName getOutputType() {
+    return outputType;
+  }
+
+  public T getValue() {
+    return value;
+  }
+
+  @Override
+  public boolean accept() {
+    if (value == null) {
+      return true;
+    }
+
+    switch (outputType) {
+    case BIGINT:
+      return value instanceof Long;
+    case DECIMAL:
+      return value instanceof BigDecimal;
+    case DOUBLE:
+      return value instanceof Double;
+    case FLOAT:
+      return value instanceof Float;
+    case INTEGER:
+      return value instanceof Integer;
+    case SMALLINT:
+      return value instanceof Short;
+    case TINYINT:
+      return value instanceof Byte;
+    case BOOLEAN:
+      return value instanceof Boolean;
+    case CHAR:
+      return value instanceof Character;
+    case VARCHAR:
+      return value instanceof String;
+    default:
+      throw new BeamSqlUnsupportedException(outputType.name());
+    }
+  }
+
+  @Override
+  public BeamSqlPrimitive<T> evaluate(BeamSQLRow inputRecord) {
+    return this;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/package-info.java
new file mode 100644
index 000000000000..9b0a9a72e122
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementation for operators in {@link org.apache.calcite.sql.fun.SqlStdOperatorTable}.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
index aac86d6b8706..935dae768401 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
@@ -122,7 +122,13 @@ public Pipeline compileBeamPipeline(String sqlStatement) throws Exception {
    */
   public BeamRelNode convertToBeamRel(String sqlStatement)
       throws ValidationException, RelConversionException, SqlParseException {
-    return (BeamRelNode) validateAndConvert(planner.parse(sqlStatement));
+    BeamRelNode beamRelNode;
+    try {
+      beamRelNode = (BeamRelNode) validateAndConvert(planner.parse(sqlStatement));
+    } finally {
+      planner.close();
+    }
+    return beamRelNode;
   }
 
   private RelNode validateAndConvert(SqlNode sqlNode)
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
index e457e8085fc9..708c5073608e 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
@@ -36,17 +36,17 @@
  *
  */
 public class BeamSqlRunner implements Serializable {
-  /**
-   *
-   */
-  private static final long serialVersionUID = -4708693435115005182L;
-
   private static final Logger LOG = LoggerFactory.getLogger(BeamSqlRunner.class);
 
   private SchemaPlus schema = Frameworks.createRootSchema(true);
 
   private BeamQueryPlanner planner = new BeamQueryPlanner(schema);
 
+  public BeamSqlRunner() {
+    //disable assertions in Calcite.
+    ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(false);
+  }
+
   /**
    * Add a schema.
    *
@@ -70,7 +70,6 @@ public void addTable(String tableName, BaseBeamTable table) {
    */
   public void submitQuery(String sqlString) throws Exception {
     planner.submitToRun(sqlString);
-    planner.planner.close();
   }
 
   /**
@@ -78,12 +77,10 @@ public void submitQuery(String sqlString) throws Exception {
    *
    */
   public String explainQuery(String sqlString)
-      throws ValidationException, RelConversionException, SqlParseException {
+    throws ValidationException, RelConversionException, SqlParseException {
     BeamRelNode exeTree = planner.convertToBeamRel(sqlString);
     String beamPlan = RelOptUtil.toString(exeTree);
     System.out.println(String.format("beamPlan>\n%s", beamPlan));
-
-    planner.planner.close();
     return beamPlan;
   }
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
index 10dd1bec559f..477be5a16765 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
@@ -18,7 +18,7 @@
 package org.apache.beam.dsls.sql.rel;
 
 import org.apache.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
-import org.apache.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutor;
 import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
 import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
@@ -58,7 +58,7 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
 
     PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
 
-    BeamSQLExpressionExecutor executor = new BeamSQLSpELExecutor(this);
+    BeamSQLExpressionExecutor executor = new BeamSQLFnExecutor(this);
 
     PCollection<BeamSQLRow> projectStream = upstream.apply(stageName,
         ParDo.of(new BeamSQLFilterFn(getRelTypeName(), executor)));
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
index dd731f8a15a1..7e27ab3773c3 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
@@ -20,7 +20,7 @@
 import java.util.List;
 
 import org.apache.beam.dsls.sql.interpreter.BeamSQLExpressionExecutor;
-import org.apache.beam.dsls.sql.interpreter.BeamSQLSpELExecutor;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutor;
 import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
 import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
@@ -69,7 +69,7 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
 
     PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
 
-    BeamSQLExpressionExecutor executor = new BeamSQLSpELExecutor(this);
+    BeamSQLExpressionExecutor executor = new BeamSQLFnExecutor(this);
 
     PCollection<BeamSQLRow> projectStream = upstream.apply(stageName, ParDo
         .of(new BeamSQLProjectFn(getRelTypeName(), executor, BeamSQLRecordType.from(rowType))));
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
index 81829e9c273f..2ecfa38f70cd 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
@@ -37,11 +37,6 @@
  * Each IO in Beam has one table schema, by extending {@link BaseBeamTable}.
  */
 public abstract class BaseBeamTable implements ScannableTable, Serializable {
-
-  /**
-   *
-   */
-  private static final long serialVersionUID = -1262988061830914193L;
   private RelDataType relDataType;
 
   protected BeamSQLRecordType beamSqlRecordType;
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
index 661b155cc85a..e4013bca6720 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
@@ -30,10 +30,6 @@
  */
 //@DefaultCoder(BeamSQLRecordTypeCoder.class)
 public class BeamSQLRecordType implements Serializable {
-  /**
-   *
-   */
-  private static final long serialVersionUID = -5318734648766104712L;
   private List<String> fieldsName = new ArrayList<>();
   private List<SqlTypeName> fieldsType = new ArrayList<>();
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
index ec330f12352a..b88a19557280 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
@@ -54,35 +54,34 @@ public void encode(BeamSQLRecordType value, OutputStream outStream,
     for (SqlTypeName fieldType : value.getFieldsType()) {
       stringCoder.encode(fieldType.name(), outStream, nested);
     }
-    outStream.flush();
+    //add a dummy field to indicate the end of record
+    intCoder.encode(value.size(), outStream, context);
   }
 
   @Override
   public BeamSQLRecordType decode(InputStream inStream,
       org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
     BeamSQLRecordType typeRecord = new BeamSQLRecordType();
-    Context nested = context.nested();
-    int size = intCoder.decode(inStream, nested);
+    int size = intCoder.decode(inStream, context.nested());
     for (int idx = 0; idx < size; ++idx) {
-      typeRecord.getFieldsName().add(stringCoder.decode(inStream, nested));
+      typeRecord.getFieldsName().add(stringCoder.decode(inStream, context.nested()));
     }
     for (int idx = 0; idx < size; ++idx) {
-      typeRecord.getFieldsType().add(SqlTypeName.valueOf(stringCoder.decode(inStream, nested)));
+      typeRecord.getFieldsType().add(
+          SqlTypeName.valueOf(stringCoder.decode(inStream, context.nested())));
     }
+    intCoder.decode(inStream, context);
     return typeRecord;
   }
 
   @Override
   public List<? extends Coder<?>> getCoderArguments() {
-    // TODO Auto-generated method stub
     return null;
   }
 
   @Override
   public void verifyDeterministic()
       throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
-    // TODO Auto-generated method stub
-
   }
 
 }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
index b65e23b5ebf5..f9dab8a41920 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
@@ -21,6 +21,7 @@
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import org.apache.beam.dsls.sql.exception.InvalidFieldException;
 import org.apache.calcite.sql.type.SqlTypeName;
 
 /**
@@ -28,10 +29,6 @@
  *
  */
 public class BeamSQLRow implements Serializable {
-  /**
-   *
-   */
-  private static final long serialVersionUID = 4569220242480160895L;
 
   private List<Integer> nullFields = new ArrayList<>();
   private List<Object> dataValues;
@@ -42,12 +39,15 @@ public BeamSQLRow(BeamSQLRecordType dataType) {
     this.dataValues = new ArrayList<>();
     for (int idx = 0; idx < dataType.size(); ++idx) {
       dataValues.add(null);
+      nullFields.add(idx);
     }
   }
 
   public BeamSQLRow(BeamSQLRecordType dataType, List<Object> dataValues) {
-    this.dataValues = dataValues;
-    this.dataType = dataType;
+    this(dataType);
+    for (int idx = 0; idx < dataValues.size(); ++idx) {
+      addField(idx, dataValues.get(idx));
+    }
   }
 
   public void addField(String fieldName, Object fieldValue) {
@@ -56,19 +56,29 @@ public void addField(String fieldName, Object fieldValue) {
 
   public void addField(int index, Object fieldValue) {
     if (fieldValue == null) {
-      dataValues.set(index, fieldValue);
-      if (!nullFields.contains(index)) {
-        nullFields.add(index);
-      }
       return;
+    } else {
+      if (nullFields.contains(index)) {
+        nullFields.remove(nullFields.indexOf(index));
+      }
     }
 
     SqlTypeName fieldType = dataType.getFieldsType().get(index);
     switch (fieldType) {
     case INTEGER:
+      if (!(fieldValue instanceof Integer)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
     case SMALLINT:
+      if (!(fieldValue instanceof Short)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
     case TINYINT:
-      if (!(fieldValue instanceof Integer)) {
+      if (!(fieldValue instanceof Byte)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
@@ -97,24 +107,24 @@ public void addField(int index, Object fieldValue) {
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
-    case TIME:
-    case TIMESTAMP:
-      if (!(fieldValue instanceof Date)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
     default:
       throw new UnsupportedDataTypeException(fieldType);
     }
     dataValues.set(index, fieldValue);
   }
 
+  public short getShort(int idx) {
+    return (Short) getFieldValue(idx);
+  }
 
   public int getInteger(int idx) {
     return (Integer) getFieldValue(idx);
   }
 
+  public float getFloat(int idx) {
+    return (Float) getFieldValue(idx);
+  }
+
   public double getDouble(int idx) {
     return (Double) getFieldValue(idx);
   }
@@ -145,47 +155,51 @@ public Object getFieldValue(int fieldIdx) {
 
     switch (fieldType) {
     case INTEGER:
+      if (!(fieldValue instanceof Integer)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
+        return fieldValue;
+      }
     case SMALLINT:
+      if (!(fieldValue instanceof Short)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
+        return fieldValue;
+      }
     case TINYINT:
-      if (!(fieldValue instanceof Integer)) {
+      if (!(fieldValue instanceof Byte)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       } else {
-        return Integer.valueOf(fieldValue.toString());
+        return fieldValue;
       }
     case DOUBLE:
       if (!(fieldValue instanceof Double)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       } else {
-        return Double.valueOf(fieldValue.toString());
+        return fieldValue;
       }
     case BIGINT:
       if (!(fieldValue instanceof Long)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       } else {
-        return Long.valueOf(fieldValue.toString());
+        return fieldValue;
       }
     case FLOAT:
       if (!(fieldValue instanceof Float)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       } else {
-        return Float.valueOf(fieldValue.toString());
+        return fieldValue;
       }
     case VARCHAR:
       if (!(fieldValue instanceof String)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue.toString();
-      }
-    case TIME:
-    case TIMESTAMP:
-      if (!(fieldValue instanceof Date)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       } else {
         return fieldValue;
       }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index 1979a004b97e..22ffaadf8b66 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -20,7 +20,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.Date;
 import java.util.List;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
@@ -55,10 +54,8 @@ public static BeamSqlRowCoder of() {
   @Override
   public void encode(BeamSQLRow value, OutputStream outStream,
       org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
-    recordTypeCoder.encode(value.getDataType(), outStream, context);
-    listCoder.encode(value.getNullFields(), outStream, context);
-
-    Context nested = context.nested();
+    recordTypeCoder.encode(value.getDataType(), outStream, context.nested());
+    listCoder.encode(value.getNullFields(), outStream, context.nested());
 
     for (int idx = 0; idx < value.size(); ++idx) {
       if (value.getNullFields().contains(idx)) {
@@ -67,36 +64,38 @@ public void encode(BeamSQLRow value, OutputStream outStream,
 
       switch (value.getDataType().getFieldsType().get(idx)) {
       case INTEGER:
+        intCoder.encode(value.getInteger(idx), outStream, context.nested());
+        break;
       case SMALLINT:
       case TINYINT:
-        intCoder.encode(value.getInteger(idx), outStream, nested);
+        intCoder.encode((int) value.getShort(idx), outStream, context.nested());
         break;
       case DOUBLE:
+        doubleCoder.encode(value.getDouble(idx), outStream, context.nested());
+        break;
       case FLOAT:
-        doubleCoder.encode(value.getDouble(idx), outStream, nested);
+        doubleCoder.encode((double) value.getFloat(idx), outStream, context.nested());
         break;
       case BIGINT:
-        longCoder.encode(value.getLong(idx), outStream, nested);
+        longCoder.encode(value.getLong(idx), outStream, context.nested());
         break;
       case VARCHAR:
-        stringCoder.encode(value.getString(idx), outStream, nested);
-        break;
-      case TIME:
-      case TIMESTAMP:
-        longCoder.encode(value.getDate(idx).getTime(), outStream, nested);
+        stringCoder.encode(value.getString(idx), outStream, context.nested());
         break;
 
       default:
         throw new UnsupportedDataTypeException(value.getDataType().getFieldsType().get(idx));
       }
     }
+    //add a dummy field to indicate the end of record
+    intCoder.encode(value.size(), outStream, context);
   }
 
   @Override
   public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
       throws CoderException, IOException {
-    BeamSQLRecordType type = recordTypeCoder.decode(inStream, context);
-    List<Integer> nullFields = listCoder.decode(inStream, context);
+    BeamSQLRecordType type = recordTypeCoder.decode(inStream, context.nested());
+    List<Integer> nullFields = listCoder.decode(inStream, context.nested());
 
     BeamSQLRow record = new BeamSQLRow(type);
     record.setNullFields(nullFields);
@@ -108,29 +107,32 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
 
       switch (type.getFieldsType().get(idx)) {
       case INTEGER:
+        record.addField(idx, intCoder.decode(inStream, context.nested()));
+        break;
       case SMALLINT:
+        record.addField(idx, intCoder.decode(inStream, context.nested()).shortValue());
+        break;
       case TINYINT:
-        record.addField(idx, intCoder.decode(inStream, context));
+        record.addField(idx, intCoder.decode(inStream, context.nested()).byteValue());
         break;
       case DOUBLE:
+        record.addField(idx, doubleCoder.decode(inStream, context.nested()));
+        break;
       case FLOAT:
-        record.addField(idx, doubleCoder.decode(inStream, context));
+        record.addField(idx, doubleCoder.decode(inStream, context.nested()).floatValue());
         break;
       case BIGINT:
-        record.addField(idx, longCoder.decode(inStream, context));
+        record.addField(idx, longCoder.decode(inStream, context.nested()));
         break;
       case VARCHAR:
-        record.addField(idx, stringCoder.decode(inStream, context));
-        break;
-      case TIME:
-      case TIMESTAMP:
-        record.addField(idx, new Date(longCoder.decode(inStream, context)));
+        record.addField(idx, stringCoder.decode(inStream, context.nested()));
         break;
 
       default:
         throw new UnsupportedDataTypeException(type.getFieldsType().get(idx));
       }
     }
+    intCoder.decode(inStream, context);
 
     return record;
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
index 915a5cc15499..0f40f33403de 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
@@ -35,11 +35,6 @@
  */
 public class BeamKafkaCSVTable extends BeamKafkaTable {
 
-  /**
-   *
-   */
-  private static final long serialVersionUID = 4754022536543333984L;
-
   public static final String DELIMITER = ",";
   private static final Logger LOG = LoggerFactory.getLogger(BeamKafkaCSVTable.class);
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
index adf46218623e..c8c851c17d2a 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
@@ -40,11 +40,6 @@
  */
 public abstract class BeamKafkaTable extends BaseBeamTable implements Serializable {
 
-  /**
-   *
-   */
-  private static final long serialVersionUID = -634715473399906527L;
-
   private String bootstrapServers;
   private List<String> topics;
   private Map<String, Object> configUpdates;
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLFilterFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLFilterFn.java
index 55086e2d9619..2ab630187d15 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLFilterFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLFilterFn.java
@@ -28,10 +28,6 @@
  *
  */
 public class BeamSQLFilterFn extends DoFn<BeamSQLRow, BeamSQLRow> {
-  /**
-   *
-   */
-  private static final long serialVersionUID = -1256111753670606705L;
 
   private String stepName;
   private BeamSQLExpressionExecutor executor;
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
index 92ebff26c25e..c146ea524d02 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLOutputToConsoleFn.java
@@ -25,10 +25,6 @@
  *
  */
 public class BeamSQLOutputToConsoleFn extends DoFn<BeamSQLRow, Void> {
-  /**
-   *
-   */
-  private static final long serialVersionUID = -1256111753670606705L;
 
   private String stepName;
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
index bafdd17f97b1..d0180571c24e 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
@@ -30,11 +30,6 @@
  *
  */
 public class BeamSQLProjectFn extends DoFn<BeamSQLRow, BeamSQLRow> {
-
-  /**
-   *
-   */
-  private static final long serialVersionUID = -1046605249999014608L;
   private String stepName;
   private BeamSQLExpressionExecutor executor;
   private BeamSQLRecordType outputRecordType;
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
new file mode 100644
index 000000000000..abbe3f790711
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlAndExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlEqualExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlInputRefExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlLessThanEqualExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.rel.BeamFilterRel;
+import org.apache.beam.dsls.sql.rel.BeamProjectRel;
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit test cases for {@link BeamSQLFnExecutor}.
+ */
+public class BeamSQLFnExecutorTest extends BeamSQLFnExecutorTestBase {
+
+  @Test
+  public void testBeamFilterRel() {
+    RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.AND,
+        Arrays.asList(
+            rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
+                Arrays.asList(rexBuilder.makeInputRef(relDataType, 0),
+                    rexBuilder.makeBigintLiteral(new BigDecimal(1000L)))),
+            rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+                Arrays.asList(rexBuilder.makeInputRef(relDataType, 1),
+                    rexBuilder.makeExactLiteral(new BigDecimal(0))))));
+
+    BeamFilterRel beamFilterRel = new BeamFilterRel(cluster, RelTraitSet.createEmpty(), null,
+        condition);
+
+    BeamSQLFnExecutor executor = new BeamSQLFnExecutor(beamFilterRel);
+    executor.prepare();
+
+    Assert.assertEquals(1, executor.exps.size());
+
+    BeamSqlExpression l1Exp = executor.exps.get(0);
+    Assert.assertTrue(l1Exp instanceof BeamSqlAndExpression);
+    Assert.assertEquals(SqlTypeName.BOOLEAN, l1Exp.getOutputType());
+
+    Assert.assertEquals(2, l1Exp.getOperands().size());
+    BeamSqlExpression l1Left = (BeamSqlExpression) l1Exp.getOperands().get(0);
+    BeamSqlExpression l1Right = (BeamSqlExpression) l1Exp.getOperands().get(1);
+
+    Assert.assertTrue(l1Left instanceof BeamSqlLessThanEqualExpression);
+    Assert.assertTrue(l1Right instanceof BeamSqlEqualExpression);
+
+    Assert.assertEquals(2, l1Left.getOperands().size());
+    BeamSqlExpression l1LeftLeft = (BeamSqlExpression) l1Left.getOperands().get(0);
+    BeamSqlExpression l1LeftRight = (BeamSqlExpression) l1Left.getOperands().get(1);
+    Assert.assertTrue(l1LeftLeft instanceof BeamSqlInputRefExpression);
+    Assert.assertTrue(l1LeftRight instanceof BeamSqlPrimitive);
+
+    Assert.assertEquals(2, l1Right.getOperands().size());
+    BeamSqlExpression l1RightLeft = (BeamSqlExpression) l1Right.getOperands().get(0);
+    BeamSqlExpression l1RightRight = (BeamSqlExpression) l1Right.getOperands().get(1);
+    Assert.assertTrue(l1RightLeft instanceof BeamSqlInputRefExpression);
+    Assert.assertTrue(l1RightRight instanceof BeamSqlPrimitive);
+  }
+
+  @Test
+  public void testBeamProjectRel() {
+    BeamRelNode relNode = new BeamProjectRel(cluster, RelTraitSet.createEmpty(),
+        relBuilder.values(relDataType, 1234567L, 0, 8.9, null).build(),
+        rexBuilder.identityProjects(relDataType), relDataType);
+    BeamSQLFnExecutor executor = new BeamSQLFnExecutor(relNode);
+
+    executor.prepare();
+    Assert.assertEquals(4, executor.exps.size());
+    Assert.assertTrue(executor.exps.get(0) instanceof BeamSqlInputRefExpression);
+    Assert.assertTrue(executor.exps.get(1) instanceof BeamSqlInputRefExpression);
+    Assert.assertTrue(executor.exps.get(2) instanceof BeamSqlInputRefExpression);
+    Assert.assertTrue(executor.exps.get(3) instanceof BeamSqlInputRefExpression);
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTestBase.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTestBase.java
new file mode 100644
index 000000000000..bfc7366232cb
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTestBase.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.dsls.sql.planner.BeamRelDataTypeSystem;
+import org.apache.beam.dsls.sql.planner.BeamRuleSets;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.volcano.VolcanoPlanner;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelBuilder;
+import org.junit.BeforeClass;
+
+/**
+ * base class to test {@link BeamSQLFnExecutor} and subclasses of {@link BeamSqlExpression}.
+ */
+public class BeamSQLFnExecutorTestBase {
+  public static RexBuilder rexBuilder = new RexBuilder(BeamQueryPlanner.TYPE_FACTORY);
+  public static RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), rexBuilder);
+
+  public static final JavaTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(
+      RelDataTypeSystem.DEFAULT);
+  public static RelDataType relDataType;
+
+  public static BeamSQLRecordType beamRecordType;
+  public static BeamSQLRow record;
+
+  public static RelBuilder relBuilder;
+
+  @BeforeClass
+  public static void prepare() {
+    relDataType = TYPE_FACTORY.builder()
+        .add("order_id", SqlTypeName.BIGINT)
+        .add("site_id", SqlTypeName.INTEGER)
+        .add("price", SqlTypeName.DOUBLE)
+        .add("order_time", SqlTypeName.BIGINT).build();
+
+    beamRecordType = BeamSQLRecordType.from(relDataType);
+    record = new BeamSQLRow(beamRecordType);
+
+    record.addField(0, 1234567L);
+    record.addField(1, 0);
+    record.addField(2, 8.9);
+    record.addField(3, 1234567L);
+
+    SchemaPlus schema = Frameworks.createRootSchema(true);
+    final List<RelTraitDef> traitDefs = new ArrayList<RelTraitDef>();
+    traitDefs.add(ConventionTraitDef.INSTANCE);
+    traitDefs.add(RelCollationTraitDef.INSTANCE);
+    FrameworkConfig config = Frameworks.newConfigBuilder()
+        .parserConfig(SqlParser.configBuilder().setLex(Lex.MYSQL).build()).defaultSchema(schema)
+        .traitDefs(traitDefs).context(Contexts.EMPTY_CONTEXT).ruleSets(BeamRuleSets.getRuleSets())
+        .costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM).build();
+
+    relBuilder = RelBuilder.create(config);
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamNullExperssionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamNullExperssionTest.java
new file mode 100644
index 000000000000..a328c884b2f1
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamNullExperssionTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlIsNullExpression} and
+ * {@link BeamSqlIsNotNullExpression}.
+ */
+public class BeamNullExperssionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test
+  public void testIsNull() {
+    BeamSqlIsNullExpression exp1 = new BeamSqlIsNullExpression(
+        new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0));
+    Assert.assertEquals(false, exp1.evaluate(record).getValue());
+
+    BeamSqlIsNullExpression exp2 = new BeamSqlIsNullExpression(
+        BeamSqlPrimitive.of(SqlTypeName.BIGINT, null));
+    Assert.assertEquals(true, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testIsNotNull() {
+    BeamSqlIsNotNullExpression exp1 = new BeamSqlIsNotNullExpression(
+        new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0));
+    Assert.assertEquals(true, exp1.evaluate(record).getValue());
+
+    BeamSqlIsNotNullExpression exp2 = new BeamSqlIsNotNullExpression(
+        BeamSqlPrimitive.of(SqlTypeName.BIGINT, null));
+    Assert.assertEquals(false, exp2.evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndOrExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndOrExpressionTest.java
new file mode 100644
index 000000000000..9dabcdc5d04b
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlAndOrExpressionTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlAndExpression}, {@link BeamSqlOrExpression}.
+ */
+public class BeamSqlAndOrExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test
+  public void testAnd() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+
+    Assert.assertTrue(new BeamSqlAndExpression(operands).evaluate(record).getValue());
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+
+    Assert.assertFalse(new BeamSqlAndExpression(operands).evaluate(record).getValue());
+  }
+
+  @Test
+  public void testOr() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+
+    Assert.assertFalse(new BeamSqlOrExpression(operands).evaluate(record).getValue());
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+
+    Assert.assertTrue(new BeamSqlOrExpression(operands).evaluate(record).getValue());
+
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
new file mode 100644
index 000000000000..b88de719b04f
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.Arrays;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for the collections of {@link BeamSqlCompareExpression}.
+ */
+public class BeamSqlCompareExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test
+  public void testEqual() {
+    BeamSqlEqualExpression exp1 = new BeamSqlEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 100L)));
+    Assert.assertEquals(false, exp1.evaluate(record).getValue());
+
+    BeamSqlEqualExpression exp2 = new BeamSqlEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+    Assert.assertEquals(true, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testLargerThan(){
+    BeamSqlLargerThanExpression exp1 = new BeamSqlLargerThanExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+    Assert.assertEquals(false, exp1.evaluate(record).getValue());
+
+    BeamSqlLargerThanExpression exp2 = new BeamSqlLargerThanExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234566L)));
+    Assert.assertEquals(true, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testLargerThanEqual(){
+    BeamSqlLargerThanEqualExpression exp1 = new BeamSqlLargerThanEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+    Assert.assertEquals(true, exp1.evaluate(record).getValue());
+
+    BeamSqlLargerThanEqualExpression exp2 = new BeamSqlLargerThanEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234568L)));
+    Assert.assertEquals(false, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testLessThan(){
+    BeamSqlLessThanExpression exp1 = new BeamSqlLessThanExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1),
+            BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1)));
+    Assert.assertEquals(true, exp1.evaluate(record).getValue());
+
+    BeamSqlLessThanExpression exp2 = new BeamSqlLessThanExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1),
+            BeamSqlPrimitive.of(SqlTypeName.INTEGER, -1)));
+    Assert.assertEquals(false, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testLessThanEqual(){
+    BeamSqlLessThanEqualExpression exp1 = new BeamSqlLessThanEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.DOUBLE, 2),
+            BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 8.9)));
+    Assert.assertEquals(true, exp1.evaluate(record).getValue());
+
+    BeamSqlLessThanEqualExpression exp2 = new BeamSqlLessThanEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.DOUBLE, 2),
+            BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 8.0)));
+    Assert.assertEquals(false, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testNotEqual(){
+    BeamSqlNotEqualExpression exp1 = new BeamSqlNotEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 3),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+    Assert.assertEquals(false, exp1.evaluate(record).getValue());
+
+    BeamSqlNotEqualExpression exp2 = new BeamSqlNotEqualExpression(
+        Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 3),
+            BeamSqlPrimitive.of(SqlTypeName.BIGINT, 0L)));
+    Assert.assertEquals(true, exp2.evaluate(record).getValue());
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpressionTest.java
new file mode 100644
index 000000000000..1cadeb0c5cb8
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpressionTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlInputRefExpression}.
+ */
+public class BeamSqlInputRefExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test
+  public void testRefInRange() {
+    BeamSqlInputRefExpression ref0 = new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0);
+    Assert.assertEquals(record.getLong(0), ref0.evaluate(record).getValue());
+
+    BeamSqlInputRefExpression ref1 = new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1);
+    Assert.assertEquals(record.getInteger(1), ref1.evaluate(record).getValue());
+
+    BeamSqlInputRefExpression ref2 = new BeamSqlInputRefExpression(SqlTypeName.DOUBLE, 2);
+    Assert.assertEquals(record.getDouble(2), ref2.evaluate(record).getValue());
+
+    BeamSqlInputRefExpression ref3 = new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 3);
+    Assert.assertEquals(record.getLong(3), ref3.evaluate(record).getValue());
+  }
+
+
+  @Test(expected = IndexOutOfBoundsException.class)
+  public void testRefOutOfRange(){
+    BeamSqlInputRefExpression ref = new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 4);
+    ref.evaluate(record).getValue();
+  }
+
+  @Test(expected = BeamInvalidOperatorException.class)
+  public void testTypeUnMatch(){
+    BeamSqlInputRefExpression ref = new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 0);
+    ref.evaluate(record).getValue();
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitiveTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitiveTest.java
new file mode 100644
index 000000000000..adb8de922f9e
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitiveTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for {@link BeamSqlPrimitive}.
+ *
+ */
+public class BeamSqlPrimitiveTest extends BeamSQLFnExecutorTestBase {
+
+  @Test
+  public void testPrimitiveInt(){
+    BeamSqlPrimitive<Integer> expInt = BeamSqlPrimitive.of(SqlTypeName.INTEGER, 100);
+    Assert.assertEquals(expInt.getValue(), expInt.evaluate(record).getValue());
+  }
+
+  @Test(expected = BeamInvalidOperatorException.class)
+  public void testPrimitiveTypeUnMatch1(){
+    BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.INTEGER, 100L);
+    Assert.assertEquals(expInt.getValue(), expInt.evaluate(record).getValue());
+  }
+  @Test(expected = BeamInvalidOperatorException.class)
+  public void testPrimitiveTypeUnMatch2(){
+    BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.DECIMAL, 100L);
+    Assert.assertEquals(expInt.getValue(), expInt.evaluate(record).getValue());
+  }
+  @Test(expected = BeamInvalidOperatorException.class)
+  public void testPrimitiveTypeUnMatch3(){
+    BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.FLOAT, 100L);
+    Assert.assertEquals(expInt.getValue(), expInt.evaluate(record).getValue());
+  }
+  @Test(expected = BeamInvalidOperatorException.class)
+  public void testPrimitiveTypeUnMatch4(){
+    BeamSqlPrimitive expInt = BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 100L);
+    Assert.assertEquals(expInt.getValue(), expInt.evaluate(record).getValue());
+  }
+
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
index 733b056d4d70..625fb719233a 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
@@ -28,6 +28,7 @@
 import org.apache.calcite.rel.type.RelProtoDataType;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 /**
@@ -35,21 +36,33 @@
  *
  */
 public class BasePlanner {
-  public static BeamSqlRunner runner = new BeamSqlRunner();
+  public static BeamSqlRunner runner;
 
   @BeforeClass
   public static void prepare() {
+    runner = new BeamSqlRunner();
+
     runner.addTable("ORDER_DETAILS", getTable());
     runner.addTable("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
     runner.addTable("SUB_ORDER_RAM", getTable());
   }
 
+  @AfterClass
+  public static void close(){
+    runner = null;
+  }
+
   private static BaseBeamTable getTable() {
     final RelProtoDataType protoRowType = new RelProtoDataType() {
       @Override
       public RelDataType apply(RelDataTypeFactory a0) {
-        return a0.builder().add("order_id", SqlTypeName.BIGINT).add("site_id", SqlTypeName.INTEGER)
-            .add("price", SqlTypeName.DOUBLE).add("order_time", SqlTypeName.TIMESTAMP).build();
+        return a0.builder()
+            .add("order_id", SqlTypeName.BIGINT)
+            .add("site_id", SqlTypeName.INTEGER)
+            .add("price", SqlTypeName.DOUBLE)
+            .add("shipping", SqlTypeName.FLOAT)
+            .add("notes", SqlTypeName.VARCHAR)
+            .build();
       }
     };
 
@@ -60,8 +73,13 @@ public static BaseBeamTable getTable(String bootstrapServer, String topic) {
     final RelProtoDataType protoRowType = new RelProtoDataType() {
       @Override
       public RelDataType apply(RelDataTypeFactory a0) {
-        return a0.builder().add("order_id", SqlTypeName.BIGINT).add("site_id", SqlTypeName.INTEGER)
-            .add("price", SqlTypeName.DOUBLE).add("order_time", SqlTypeName.TIMESTAMP).build();
+        return a0.builder()
+            .add("order_id", SqlTypeName.BIGINT)
+            .add("site_id", SqlTypeName.INTEGER)
+            .add("price", SqlTypeName.DOUBLE)
+            .add("shipping", SqlTypeName.FLOAT)
+            .add("notes", SqlTypeName.VARCHAR)
+            .build();
       }
     };
 
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
index 9dde0f1e5fb7..5d1052b55930 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
@@ -32,7 +32,7 @@ public void selectAll() throws Exception {
     String plan = runner.explainQuery(sql);
 
     String expectedPlan =
-        "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], order_time=[$3])\n"
+        "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], shipping=[$3], notes=[$4])\n"
         + "  BeamIOSourceRel(table=[[ORDER_DETAILS]])\n";
     Assert.assertEquals("explain doesn't match", expectedPlan, plan);
   }
@@ -58,7 +58,8 @@ public void insertSelectFilter() throws Exception {
 
     String expectedPlan =
         "BeamIOSinkRel(table=[[SUB_ORDER]], operation=[INSERT], flattened=[true])\n"
-        + "  BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], order_time=[null])\n"
+        + "  BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], shipping=[null],"
+            + " notes=[null])\n"
         + "    BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2])\n"
         + "      BeamFilterRel(condition=[AND(=($1, 0), >($2, 20))])\n"
         + "        BeamIOSourceRel(table=[[ORDER_DETAILS]])\n";
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
index d32b19b3133c..1ca9eb33994d 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
@@ -17,7 +17,6 @@
  */
 package org.apache.beam.dsls.sql.planner;
 
-import org.apache.beam.sdk.Pipeline;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -30,13 +29,11 @@ public class BeamPlannerSubmitTest extends BasePlanner {
   public void insertSelectFilter() throws Exception {
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
         + " order_id, site_id, price " + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 and price > 20";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
-    runner.getPlanner().planner.close();
 
-    pipeline.run().waitUntilFinish();
+    runner.submitQuery(sql);
 
     Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
-    Assert.assertEquals("order_id=12345,site_id=0,price=20.5,order_time=null",
+    Assert.assertEquals("order_id=12345,site_id=0,price=20.5,shipping=null,notes=null",
         MockedBeamSQLTable.CONTENT.get(0));
   }
 
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
index 8631a6e5a479..538607f5d6cb 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
@@ -18,7 +18,6 @@
 package org.apache.beam.dsls.sql.planner;
 
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
@@ -39,11 +38,6 @@
  */
 public class MockedBeamSQLTable extends BaseBeamTable {
 
-  /**
-   *
-   */
-  private static final long serialVersionUID = 1373168368414036932L;
-
   public static final List<String> CONTENT = new ArrayList<>();
 
   public MockedBeamSQLTable(RelProtoDataType protoRowType) {
@@ -61,25 +55,25 @@ public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
     row1.addField(0, 12345L);
     row1.addField(1, 0);
     row1.addField(2, 10.5);
-    row1.addField(3, new Date());
+    row1.addField(3, 123.4f);
 
     BeamSQLRow row2 = new BeamSQLRow(beamSqlRecordType);
     row2.addField(0, 12345L);
     row2.addField(1, 1);
     row2.addField(2, 20.5);
-    row2.addField(3, new Date());
+    row2.addField(3, 234.5f);
 
     BeamSQLRow row3 = new BeamSQLRow(beamSqlRecordType);
     row3.addField(0, 12345L);
     row3.addField(1, 0);
     row3.addField(2, 20.5);
-    row3.addField(3, new Date());
+    row3.addField(3, 345.6f);
 
     BeamSQLRow row4 = new BeamSQLRow(beamSqlRecordType);
     row4.addField(0, null);
     row4.addField(1, null);
     row4.addField(2, 20.5);
-    row4.addField(3, new Date());
+    row4.addField(3, 456.7f);
 
     return Create.of(row1, row2, row3);
   }

From e3185b0088dedec47a32383ae8114444104f4743 Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Thu, 4 May 2017 11:44:14 +0800
Subject: [PATCH 370/387] Fix inconsistent mapping for SQL FLOAT

---
 dsls/sql/pom.xml                              |  9 +-
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java | 93 ++++++++++---------
 .../dsls/sql/schema/BeamSqlRowCoderTest.java  | 63 +++++++++++++
 3 files changed, 117 insertions(+), 48 deletions(-)
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java

diff --git a/dsls/sql/pom.xml b/dsls/sql/pom.xml
index e2f09beb1ac1..6139adafb4ed 100644
--- a/dsls/sql/pom.xml
+++ b/dsls/sql/pom.xml
@@ -24,7 +24,7 @@
     <artifactId>beam-dsls-parent</artifactId>
     <version>0.7.0-SNAPSHOT</version>
   </parent>
-  
+
   <artifactId>beam-dsls-sql</artifactId>
   <name>Apache Beam :: DSLs :: SQL</name>
   <description>Beam SQL provides a new interface to generate a Beam pipeline from SQL statement</description>
@@ -36,7 +36,7 @@
     <maven.build.timestamp.format>yyyy-MM-dd HH:mm</maven.build.timestamp.format>
     <calcite-version>1.11.0</calcite-version>
   </properties>
-  
+
   <build>
     <resources>
       <resource>
@@ -199,5 +199,10 @@
       <artifactId>calcite-linq4j</artifactId>
       <version>${calcite-version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index 22ffaadf8b66..9b2474a334f1 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -63,28 +63,29 @@ public void encode(BeamSQLRow value, OutputStream outStream,
       }
 
       switch (value.getDataType().getFieldsType().get(idx)) {
-      case INTEGER:
-        intCoder.encode(value.getInteger(idx), outStream, context.nested());
-        break;
-      case SMALLINT:
-      case TINYINT:
-        intCoder.encode((int) value.getShort(idx), outStream, context.nested());
-        break;
-      case DOUBLE:
-        doubleCoder.encode(value.getDouble(idx), outStream, context.nested());
-        break;
-      case FLOAT:
-        doubleCoder.encode((double) value.getFloat(idx), outStream, context.nested());
-        break;
-      case BIGINT:
-        longCoder.encode(value.getLong(idx), outStream, context.nested());
-        break;
-      case VARCHAR:
-        stringCoder.encode(value.getString(idx), outStream, context.nested());
-        break;
-
-      default:
-        throw new UnsupportedDataTypeException(value.getDataType().getFieldsType().get(idx));
+        case INTEGER:
+          intCoder.encode(value.getInteger(idx), outStream, context.nested());
+          break;
+        case SMALLINT:
+        case TINYINT:
+          intCoder.encode((int) value.getShort(idx), outStream, context.nested());
+          break;
+        case DOUBLE:
+          doubleCoder.encode(value.getDouble(idx), outStream, context.nested());
+          break;
+        case FLOAT:
+          doubleCoder.encode(Double.parseDouble(
+              String.valueOf(value.getFloat(idx))), outStream, context.nested());
+          break;
+        case BIGINT:
+          longCoder.encode(value.getLong(idx), outStream, context.nested());
+          break;
+        case VARCHAR:
+          stringCoder.encode(value.getString(idx), outStream, context.nested());
+          break;
+
+        default:
+          throw new UnsupportedDataTypeException(value.getDataType().getFieldsType().get(idx));
       }
     }
     //add a dummy field to indicate the end of record
@@ -106,30 +107,30 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
       }
 
       switch (type.getFieldsType().get(idx)) {
-      case INTEGER:
-        record.addField(idx, intCoder.decode(inStream, context.nested()));
-        break;
-      case SMALLINT:
-        record.addField(idx, intCoder.decode(inStream, context.nested()).shortValue());
-        break;
-      case TINYINT:
-        record.addField(idx, intCoder.decode(inStream, context.nested()).byteValue());
-        break;
-      case DOUBLE:
-        record.addField(idx, doubleCoder.decode(inStream, context.nested()));
-        break;
-      case FLOAT:
-        record.addField(idx, doubleCoder.decode(inStream, context.nested()).floatValue());
-        break;
-      case BIGINT:
-        record.addField(idx, longCoder.decode(inStream, context.nested()));
-        break;
-      case VARCHAR:
-        record.addField(idx, stringCoder.decode(inStream, context.nested()));
-        break;
-
-      default:
-        throw new UnsupportedDataTypeException(type.getFieldsType().get(idx));
+        case INTEGER:
+          record.addField(idx, intCoder.decode(inStream, context.nested()));
+          break;
+        case SMALLINT:
+          record.addField(idx, intCoder.decode(inStream, context.nested()).shortValue());
+          break;
+        case TINYINT:
+          record.addField(idx, intCoder.decode(inStream, context.nested()).byteValue());
+          break;
+        case DOUBLE:
+          record.addField(idx, doubleCoder.decode(inStream, context.nested()));
+          break;
+        case FLOAT:
+          record.addField(idx, doubleCoder.decode(inStream, context.nested()).floatValue());
+          break;
+        case BIGINT:
+          record.addField(idx, longCoder.decode(inStream, context.nested()));
+          break;
+        case VARCHAR:
+          record.addField(idx, stringCoder.decode(inStream, context.nested()));
+          break;
+
+        default:
+          throw new UnsupportedDataTypeException(type.getFieldsType().get(idx));
       }
     }
     intCoder.decode(inStream, context);
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java
new file mode 100644
index 000000000000..f2077947ec3a
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema;
+
+import org.apache.beam.sdk.testing.CoderProperties;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Tests for BeamSqlRowCoder.
+ */
+public class BeamSqlRowCoderTest {
+
+  @Test
+  public void encodeAndDecode() throws Exception {
+    final RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder()
+            .add("id", SqlTypeName.INTEGER)
+            .add("order_id", SqlTypeName.BIGINT)
+            .add("price", SqlTypeName.FLOAT)
+            .add("amount", SqlTypeName.DOUBLE)
+            .add("user_name", SqlTypeName.VARCHAR)
+            .build();
+      }
+    };
+
+    BeamSQLRecordType beamSQLRecordType = BeamSQLRecordType.from(
+        protoRowType.apply(new JavaTypeFactoryImpl(
+        RelDataTypeSystem.DEFAULT)));
+    BeamSQLRow row = new BeamSQLRow(beamSQLRecordType);
+    row.addField(0, 1);
+    row.addField(1, 1L);
+    row.addField(2, 1.1F);
+    row.addField(3, 1.1);
+    row.addField(4, "hello");
+
+    BeamSqlRowCoder coder = BeamSqlRowCoder.of();
+    CoderProperties.coderDecodeEncodeEqual(coder, row);
+  }
+}

From 6fa03c0012f3a911c0ffda9f2c3f441f90f0bafa Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Thu, 4 May 2017 22:15:02 +0800
Subject: [PATCH 371/387] [BEAM-2158] Implement the arithmetic operators

---
 .../sql/interpreter/BeamSQLFnExecutor.java    |  21 +-
 .../BeamSqlArithmeticExpression.java          | 101 +++++++
 .../arithmetic/BeamSqlDivideExpression.java   |  40 +++
 .../arithmetic/BeamSqlMinusExpression.java    |  40 +++
 .../arithmetic/BeamSqlModExpression.java      |  40 +++
 .../arithmetic/BeamSqlMultiplyExpression.java |  40 +++
 .../arithmetic/BeamSqlPlusExpression.java     |  40 +++
 .../operator/arithmetic/package-info.java     |  22 ++
 .../interpreter/BeamSQLFnExecutorTest.java    |  55 +++-
 .../BeamSqlArithmeticExpressionTest.java      | 251 ++++++++++++++++++
 10 files changed, 637 insertions(+), 13 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlDivideExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/package-info.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
index 32e2ffcb50d4..78663f8e7ec6 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -33,6 +33,11 @@
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlNotEqualExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlOrExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlDivideExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlMinusExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlModExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlMultiplyExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlPlusExpression;
 import org.apache.beam.dsls.sql.rel.BeamFilterRel;
 import org.apache.beam.dsls.sql.rel.BeamProjectRel;
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
@@ -74,7 +79,7 @@ public BeamSQLFnExecutor(BeamRelNode relNode) {
    * {@link #buildExpression(RexNode)} visits the operands of {@link RexNode} recursively,
    * and represent each {@link SqlOperator} with a corresponding {@link BeamSqlExpression}.
    */
-  private BeamSqlExpression buildExpression(RexNode rexNode) {
+  static BeamSqlExpression buildExpression(RexNode rexNode) {
     if (rexNode instanceof RexLiteral) {
       RexLiteral node = (RexLiteral) rexNode;
       return BeamSqlPrimitive.of(node.getTypeName(), node.getValue());
@@ -107,12 +112,24 @@ private BeamSqlExpression buildExpression(RexNode rexNode) {
         case "<=":
           return new BeamSqlLessThanEqualExpression(subExps);
 
+        // arithmetic operators
+        case "+":
+          return new BeamSqlPlusExpression(subExps);
+        case "-":
+          return new BeamSqlMinusExpression(subExps);
+        case "*":
+          return new BeamSqlMultiplyExpression(subExps);
+        case "/":
+          return new BeamSqlDivideExpression(subExps);
+        case "MOD":
+          return new BeamSqlModExpression(subExps);
+
         case "IS NULL":
           return new BeamSqlIsNullExpression(subExps.get(0));
         case "IS NOT NULL":
           return new BeamSqlIsNotNullExpression(subExps.get(0));
       default:
-        throw new BeamSqlUnsupportedException();
+        throw new BeamSqlUnsupportedException("Operator: " + opName + " not supported yet!");
       }
     } else {
       throw new BeamSqlUnsupportedException(
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java
new file mode 100644
index 000000000000..5e1d068ff34c
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpression.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Base class for all arithmetic operators.
+ */
+public abstract class BeamSqlArithmeticExpression extends BeamSqlExpression {
+  private BeamSqlArithmeticExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  public BeamSqlArithmeticExpression(List<BeamSqlExpression> operands) {
+    // the outputType can not be determined in constructor
+    // will be determined in evaluate() method. ANY here is just a placeholder.
+    super(operands, SqlTypeName.ANY);
+  }
+
+  @Override public boolean accept() {
+    if (operands.size() != 2) {
+      return false;
+    }
+
+    for (BeamSqlExpression operand : operands) {
+      if (!SqlTypeName.NUMERIC_TYPES.contains(operand.getOutputType())) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * https://dev.mysql.com/doc/refman/5.7/en/arithmetic-functions.html.
+   */
+  @Override public BeamSqlPrimitive<? extends Number> evaluate(BeamSQLRow inputRecord) {
+    BeamSqlExpression leftOp = operands.get(0);
+    BeamSqlExpression rightOp = operands.get(1);
+
+    // In the case of -, +, and *, the result is calculated as Long if both
+    // operands are INT_TYPES(byte, short, integer, long).
+    if (SqlTypeName.INT_TYPES.contains(leftOp.getOutputType())
+        && SqlTypeName.INT_TYPES.contains(rightOp.getOutputType())) {
+      Long leftValue = Long.valueOf(leftOp.evaluate(inputRecord).getValue().toString());
+      Long rightValue = Long.valueOf(rightOp.evaluate(inputRecord).getValue().toString());
+      Long ret = calc(leftValue, rightValue);
+      return BeamSqlPrimitive.of(SqlTypeName.BIGINT, ret);
+    } else {
+      // If any of the operands of a +, -, /, *, % is a real
+      //  OR
+      // It is a division calculation
+      // we treat them as Double
+      double leftValue = getDouble(inputRecord, leftOp);
+      double rightValue = getDouble(inputRecord, rightOp);
+      return BeamSqlPrimitive.of(SqlTypeName.DOUBLE, calc(leftValue, rightValue));
+    }
+  }
+
+  private double getDouble(BeamSQLRow inputRecord, BeamSqlExpression op) {
+    Object raw = op.evaluate(inputRecord).getValue();
+    Double ret = null;
+    if (SqlTypeName.NUMERIC_TYPES.contains(op.getOutputType())) {
+      ret = ((Number) raw).doubleValue();
+    }
+
+    return ret;
+  }
+
+  /**
+   * For {@link SqlTypeName#INT_TYPES} calculation of '+', '-', '*'.
+   */
+  public abstract Long calc(Long left, Long right);
+
+
+  /**
+   * For other {@link SqlTypeName#NUMERIC_TYPES} of '+', '-', '*', '/'.
+   */
+  public abstract Double calc(Number left, Number right);
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlDivideExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlDivideExpression.java
new file mode 100644
index 000000000000..c23f54c724a1
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlDivideExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '/' operator.
+ */
+public class BeamSqlDivideExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlDivideExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override public Long calc(Long left, Long right) {
+    return left / right;
+  }
+
+  @Override public Double calc(Number left, Number right) {
+    return left.doubleValue() / right.doubleValue();
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMinusExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
new file mode 100644
index 000000000000..c6d7ca04d9a2
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '-' operator.
+ */
+public class BeamSqlMinusExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlMinusExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override public Long calc(Long left, Long right) {
+    return left - right;
+  }
+
+  @Override public Double calc(Number left, Number right) {
+    return left.doubleValue() - right.doubleValue();
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java
new file mode 100644
index 000000000000..6323e95c2448
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '%' operator.
+ */
+public class BeamSqlModExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlModExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override public Long calc(Long left, Long right) {
+    return left % right;
+  }
+
+  @Override public Double calc(Number left, Number right) {
+    return left.doubleValue() % right.doubleValue();
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
new file mode 100644
index 000000000000..42ba4a5cbdd8
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '*' operator.
+ */
+public class BeamSqlMultiplyExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlMultiplyExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override public Long calc(Long left, Long right) {
+    return left * right;
+  }
+
+  @Override public Double calc(Number left, Number right) {
+    return left.doubleValue() * right.doubleValue();
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlPlusExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
new file mode 100644
index 000000000000..59be0532bfce
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '+' operator.
+ */
+public class BeamSqlPlusExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlPlusExpression(List<BeamSqlExpression> operands) {
+    super(operands);
+  }
+
+  @Override public Double calc(Number left, Number right) {
+    return left.doubleValue() + right.doubleValue();
+  }
+
+  @Override public Long calc(Long left, Long right) {
+    return left + right;
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/package-info.java
new file mode 100644
index 000000000000..b8f2175a1169
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Arithmetic operators.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
index abbe3f790711..8df0865395b1 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
@@ -17,19 +17,28 @@
  */
 package org.apache.beam.dsls.sql.interpreter;
 
+import static org.junit.Assert.assertTrue;
+
 import java.math.BigDecimal;
 import java.util.Arrays;
+
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlAndExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlEqualExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlInputRefExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlLessThanEqualExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlDivideExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlMinusExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlModExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlMultiplyExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlPlusExpression;
 import org.apache.beam.dsls.sql.rel.BeamFilterRel;
 import org.apache.beam.dsls.sql.rel.BeamProjectRel;
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.junit.Assert;
@@ -60,27 +69,27 @@ public void testBeamFilterRel() {
     Assert.assertEquals(1, executor.exps.size());
 
     BeamSqlExpression l1Exp = executor.exps.get(0);
-    Assert.assertTrue(l1Exp instanceof BeamSqlAndExpression);
+    assertTrue(l1Exp instanceof BeamSqlAndExpression);
     Assert.assertEquals(SqlTypeName.BOOLEAN, l1Exp.getOutputType());
 
     Assert.assertEquals(2, l1Exp.getOperands().size());
     BeamSqlExpression l1Left = (BeamSqlExpression) l1Exp.getOperands().get(0);
     BeamSqlExpression l1Right = (BeamSqlExpression) l1Exp.getOperands().get(1);
 
-    Assert.assertTrue(l1Left instanceof BeamSqlLessThanEqualExpression);
-    Assert.assertTrue(l1Right instanceof BeamSqlEqualExpression);
+    assertTrue(l1Left instanceof BeamSqlLessThanEqualExpression);
+    assertTrue(l1Right instanceof BeamSqlEqualExpression);
 
     Assert.assertEquals(2, l1Left.getOperands().size());
     BeamSqlExpression l1LeftLeft = (BeamSqlExpression) l1Left.getOperands().get(0);
     BeamSqlExpression l1LeftRight = (BeamSqlExpression) l1Left.getOperands().get(1);
-    Assert.assertTrue(l1LeftLeft instanceof BeamSqlInputRefExpression);
-    Assert.assertTrue(l1LeftRight instanceof BeamSqlPrimitive);
+    assertTrue(l1LeftLeft instanceof BeamSqlInputRefExpression);
+    assertTrue(l1LeftRight instanceof BeamSqlPrimitive);
 
     Assert.assertEquals(2, l1Right.getOperands().size());
     BeamSqlExpression l1RightLeft = (BeamSqlExpression) l1Right.getOperands().get(0);
     BeamSqlExpression l1RightRight = (BeamSqlExpression) l1Right.getOperands().get(1);
-    Assert.assertTrue(l1RightLeft instanceof BeamSqlInputRefExpression);
-    Assert.assertTrue(l1RightRight instanceof BeamSqlPrimitive);
+    assertTrue(l1RightLeft instanceof BeamSqlInputRefExpression);
+    assertTrue(l1RightRight instanceof BeamSqlPrimitive);
   }
 
   @Test
@@ -92,10 +101,34 @@ public void testBeamProjectRel() {
 
     executor.prepare();
     Assert.assertEquals(4, executor.exps.size());
-    Assert.assertTrue(executor.exps.get(0) instanceof BeamSqlInputRefExpression);
-    Assert.assertTrue(executor.exps.get(1) instanceof BeamSqlInputRefExpression);
-    Assert.assertTrue(executor.exps.get(2) instanceof BeamSqlInputRefExpression);
-    Assert.assertTrue(executor.exps.get(3) instanceof BeamSqlInputRefExpression);
+    assertTrue(executor.exps.get(0) instanceof BeamSqlInputRefExpression);
+    assertTrue(executor.exps.get(1) instanceof BeamSqlInputRefExpression);
+    assertTrue(executor.exps.get(2) instanceof BeamSqlInputRefExpression);
+    assertTrue(executor.exps.get(3) instanceof BeamSqlInputRefExpression);
   }
 
+
+  @Test
+  public void testBuildExpression_arithmetic() {
+    testBuildArithmeticExpression(SqlStdOperatorTable.PLUS, BeamSqlPlusExpression.class);
+    testBuildArithmeticExpression(SqlStdOperatorTable.MINUS, BeamSqlMinusExpression.class);
+    testBuildArithmeticExpression(SqlStdOperatorTable.MULTIPLY, BeamSqlMultiplyExpression.class);
+    testBuildArithmeticExpression(SqlStdOperatorTable.DIVIDE, BeamSqlDivideExpression.class);
+    testBuildArithmeticExpression(SqlStdOperatorTable.MOD, BeamSqlModExpression.class);
+  }
+
+  private void testBuildArithmeticExpression(SqlOperator fn,
+      Class<? extends BeamSqlExpression> clazz) {
+    RexNode rexNode;
+    BeamSqlExpression exp;
+    rexNode = rexBuilder.makeCall(fn,
+        Arrays.asList(
+            rexBuilder.makeBigintLiteral(new BigDecimal(1L)),
+            rexBuilder.makeBigintLiteral(new BigDecimal(1L))
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+
+    assertTrue(exp.getClass().equals(clazz));
+  }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
new file mode 100644
index 000000000000..abebf1727531
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Tests for {@code BeamSqlArithmeticExpression}.
+ */
+public class BeamSqlArithmeticExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void testAccept_normal() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    // byte, short
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+    assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+    // integer, long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+    // float, double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+    assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+    // varchar
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "1"));
+    assertFalse(new BeamSqlPlusExpression(operands).accept());
+  }
+
+  @Test public void testAccept_exception() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    // more than 2 operands
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+    assertFalse(new BeamSqlPlusExpression(operands).accept());
+
+    // boolean
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    assertFalse(new BeamSqlPlusExpression(operands).accept());
+  }
+
+  @Test public void testPlus() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    // integer + integer => long
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertEquals(2L, new BeamSqlPlusExpression(operands).evaluate(record).getValue());
+
+    // integer + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2L, new BeamSqlPlusExpression(operands).evaluate(record).getValue());
+
+    // long + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2L, new BeamSqlPlusExpression(operands).evaluate(record).getValue());
+
+    // float + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(Double.valueOf(Double.valueOf(1.1F) + 1),
+        new BeamSqlPlusExpression(operands).evaluate(record).getValue());
+
+    // double + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2.1, new BeamSqlPlusExpression(operands).evaluate(record).getValue());
+  }
+
+  @Test public void testMinus() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    // integer + integer => long
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertEquals(1L, new BeamSqlMinusExpression(operands).evaluate(record).getValue());
+
+    // integer + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(1L, new BeamSqlMinusExpression(operands).evaluate(record).getValue());
+
+    // long + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(1L, new BeamSqlMinusExpression(operands).evaluate(record).getValue());
+
+    // float + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 2.1F));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(Double.valueOf(Double.valueOf(2.1F) - 1),
+        new BeamSqlMinusExpression(operands).evaluate(record).getValue());
+
+    // double + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(1.1, new BeamSqlMinusExpression(operands).evaluate(record).getValue());
+  }
+
+  @Test public void testMultiply() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    // integer + integer => long
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertEquals(2L, new BeamSqlMultiplyExpression(operands).evaluate(record).getValue());
+
+    // integer + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2L, new BeamSqlMultiplyExpression(operands).evaluate(record).getValue());
+
+    // long + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2L, new BeamSqlMultiplyExpression(operands).evaluate(record).getValue());
+
+    // float + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 2.1F));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(Double.valueOf(Double.valueOf(2.1F) * 1),
+        new BeamSqlMultiplyExpression(operands).evaluate(record).getValue());
+
+    // double + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2.1, new BeamSqlMultiplyExpression(operands).evaluate(record).getValue());
+  }
+
+  @Test public void testDivide() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    // integer + integer => long
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertEquals(2L, new BeamSqlDivideExpression(operands).evaluate(record).getValue());
+
+    // integer + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2L, new BeamSqlDivideExpression(operands).evaluate(record).getValue());
+
+    // long + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2L, new BeamSqlDivideExpression(operands).evaluate(record).getValue());
+
+    // float + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 2.1F));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(Double.valueOf(Double.valueOf(2.1F) / 1),
+        new BeamSqlDivideExpression(operands).evaluate(record).getValue());
+
+    // double + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 2.1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+    assertEquals(2.1, new BeamSqlDivideExpression(operands).evaluate(record).getValue());
+  }
+
+  @Test public void testMod() {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    // integer + integer => long
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    assertEquals(1L, new BeamSqlModExpression(operands).evaluate(record).getValue());
+
+    // integer + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+    assertEquals(1L, new BeamSqlModExpression(operands).evaluate(record).getValue());
+
+    // long + long => long
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 3L));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+    assertEquals(1L, new BeamSqlModExpression(operands).evaluate(record).getValue());
+
+    // float + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 3.1F));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+    assertEquals(Double.valueOf(Double.valueOf(3.1F) % 2),
+        new BeamSqlModExpression(operands).evaluate(record).getValue());
+
+    // double + long => double
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 3.1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 2L));
+    assertEquals(1.1, new BeamSqlModExpression(operands).evaluate(record).getValue());
+  }
+}

From 5cc3fb461dda26c4e038b7b7b4d1071e0027e684 Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Sat, 6 May 2017 01:20:54 +0800
Subject: [PATCH 372/387] [BEAM-2161] Add support for String operators

---
 .../sql/interpreter/BeamSQLFnExecutor.java    |  33 +++++
 .../operator/BeamSqlExpression.java           |  12 ++
 .../operator/BeamSqlPrimitive.java            |   4 +-
 .../string/BeamSqlCharLengthExpression.java   |  40 ++++++
 .../string/BeamSqlConcatExpression.java       |  63 +++++++++
 .../string/BeamSqlInitCapExpression.java      |  56 ++++++++
 .../string/BeamSqlLowerExpression.java        |  40 ++++++
 .../string/BeamSqlOverlayExpression.java      |  77 ++++++++++
 .../string/BeamSqlPositionExpression.java     |  73 ++++++++++
 .../string/BeamSqlStringUnaryExpression.java  |  45 ++++++
 .../string/BeamSqlSubstringExpression.java    |  83 +++++++++++
 .../string/BeamSqlTrimExpression.java         | 101 ++++++++++++++
 .../string/BeamSqlUpperExpression.java        |  40 ++++++
 .../operator/string/package-info.java         |  22 +++
 .../interpreter/BeamSQLFnExecutorTest.java    | 131 +++++++++++++++++-
 .../BeamSqlCharLengthExpressionTest.java      |  45 ++++++
 .../string/BeamSqlConcatExpressionTest.java   |  67 +++++++++
 .../string/BeamSqlInitCapExpressionTest.java  |  55 ++++++++
 .../string/BeamSqlLowerExpressionTest.java    |  45 ++++++
 .../string/BeamSqlOverlayExpressionTest.java  |  88 ++++++++++++
 .../string/BeamSqlPositionExpressionTest.java |  85 ++++++++++++
 .../BeamSqlStringUnaryExpressionTest.java     |  53 +++++++
 .../BeamSqlSubstringExpressionTest.java       | 102 ++++++++++++++
 .../string/BeamSqlTrimExpressionTest.java     | 102 ++++++++++++++
 .../string/BeamSqlUpperExpressionTest.java    |  45 ++++++
 25 files changed, 1501 insertions(+), 6 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/package-info.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
index 78663f8e7ec6..a14d347d27e0 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlAndExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlEqualExpression;
@@ -38,6 +39,15 @@
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlModExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlMultiplyExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlPlusExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlCharLengthExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlConcatExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlInitCapExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlLowerExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlOverlayExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlPositionExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlSubstringExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlTrimExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlUpperExpression;
 import org.apache.beam.dsls.sql.rel.BeamFilterRel;
 import org.apache.beam.dsls.sql.rel.BeamProjectRel;
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
@@ -82,6 +92,7 @@ public BeamSQLFnExecutor(BeamRelNode relNode) {
   static BeamSqlExpression buildExpression(RexNode rexNode) {
     if (rexNode instanceof RexLiteral) {
       RexLiteral node = (RexLiteral) rexNode;
+
       return BeamSqlPrimitive.of(node.getTypeName(), node.getValue());
     } else if (rexNode instanceof RexInputRef) {
       RexInputRef node = (RexInputRef) rexNode;
@@ -124,6 +135,28 @@ static BeamSqlExpression buildExpression(RexNode rexNode) {
         case "MOD":
           return new BeamSqlModExpression(subExps);
 
+        // string operators
+        case "||":
+          return new BeamSqlConcatExpression(subExps);
+        case "POSITION":
+          return new BeamSqlPositionExpression(subExps);
+        case "CHAR_LENGTH":
+        case "CHARACTER_LENGTH":
+          return new BeamSqlCharLengthExpression(subExps);
+        case "UPPER":
+          return new BeamSqlUpperExpression(subExps);
+        case "LOWER":
+          return new BeamSqlLowerExpression(subExps);
+        case "TRIM":
+          return new BeamSqlTrimExpression(subExps);
+        case "SUBSTRING":
+          return new BeamSqlSubstringExpression(subExps);
+        case "OVERLAY":
+          return new BeamSqlOverlayExpression(subExps);
+        case "INITCAP":
+          return new BeamSqlInitCapExpression(subExps);
+
+
         case "IS NULL":
           return new BeamSqlIsNullExpression(subExps.get(0));
         case "IS NOT NULL":
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
index c44795f5b692..54289e6b47fd 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
@@ -41,6 +41,18 @@ public BeamSqlExpression(List<BeamSqlExpression> operands, SqlTypeName outputTyp
     this.outputType = outputType;
   }
 
+  public BeamSqlExpression op(int idx) {
+    return operands.get(idx);
+  }
+
+  public SqlTypeName opType(int idx) {
+    return op(idx).getOutputType();
+  }
+
+  public <T> T opValueEvaluated(int idx, BeamSQLRow row) {
+    return (T) op(idx).evaluate(row).getValue();
+  }
+
   /**
    * assertion to make sure the input and output are supported in this expression.
    */
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
index 71852ffcd235..dbe6c3c0bf98 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
@@ -23,6 +23,7 @@
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.NlsString;
 
 /**
  * {@link BeamSqlPrimitive} is a special, self-reference {@link BeamSqlExpression}.
@@ -86,9 +87,8 @@ public boolean accept() {
     case BOOLEAN:
       return value instanceof Boolean;
     case CHAR:
-      return value instanceof Character;
     case VARCHAR:
-      return value instanceof String;
+      return value instanceof String || value instanceof NlsString;
     default:
       throw new BeamSqlUnsupportedException(outputType.name());
     }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpression.java
new file mode 100644
index 000000000000..7dbd7f13e87a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'CHAR_LENGTH' operator.
+ */
+public class BeamSqlCharLengthExpression extends BeamSqlStringUnaryExpression {
+  public BeamSqlCharLengthExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.INTEGER);
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String str = opValueEvaluated(0, inputRecord);
+    return BeamSqlPrimitive.of(SqlTypeName.INTEGER, str.length());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java
new file mode 100644
index 000000000000..a56e9b1e1ad3
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * String concat operator.
+ */
+public class BeamSqlConcatExpression extends BeamSqlExpression {
+
+  protected BeamSqlConcatExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  public BeamSqlConcatExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public boolean accept() {
+    if (operands.size() != 2) {
+      return false;
+    }
+
+    for (BeamSqlExpression exp : getOperands()) {
+      if (!SqlTypeName.CHAR_TYPES.contains(exp.getOutputType())) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String left = opValueEvaluated(0, inputRecord);
+    String right = opValueEvaluated(1, inputRecord);
+
+    return BeamSqlPrimitive.of(SqlTypeName.VARCHAR,
+        new StringBuilder(left.length() + right.length())
+            .append(left).append(right).toString());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java
new file mode 100644
index 000000000000..3d0125f8f323
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'INITCAP' operator.
+ */
+public class BeamSqlInitCapExpression extends BeamSqlStringUnaryExpression {
+  public BeamSqlInitCapExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String str = opValueEvaluated(0, inputRecord);
+
+    StringBuilder ret = new StringBuilder(str);
+    boolean isInit = true;
+    for (int i = 0; i < str.length(); i++) {
+      if (Character.isWhitespace(str.charAt(i))) {
+        isInit = true;
+        continue;
+      }
+
+      if (isInit) {
+        ret.setCharAt(i, Character.toUpperCase(str.charAt(i)));
+        isInit = false;
+      } else {
+        ret.setCharAt(i, Character.toLowerCase(str.charAt(i)));
+      }
+    }
+    return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, ret.toString());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpression.java
new file mode 100644
index 000000000000..1855c657134d
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'LOWER' operator.
+ */
+public class BeamSqlLowerExpression extends BeamSqlStringUnaryExpression {
+  public BeamSqlLowerExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String str = opValueEvaluated(0, inputRecord);
+    return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.toLowerCase());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java
new file mode 100644
index 000000000000..73f2591ed742
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'OVERLAY' operator.
+ *
+ * <p>
+ *   OVERLAY(string1 PLACING string2 FROM integer [ FOR integer2 ])
+ * </p>
+ */
+public class BeamSqlOverlayExpression extends BeamSqlExpression {
+  public BeamSqlOverlayExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public boolean accept() {
+    if (operands.size() < 3 || operands.size() > 4) {
+      return false;
+    }
+
+    if (!SqlTypeName.CHAR_TYPES.contains(opType(0))
+        || !SqlTypeName.CHAR_TYPES.contains(opType(1))
+        || !SqlTypeName.INT_TYPES.contains(opType(2))) {
+      return false;
+    }
+
+    if (operands.size() == 4 && !SqlTypeName.INT_TYPES.contains(opType(3))) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String str = opValueEvaluated(0, inputRecord);
+    String replaceStr = opValueEvaluated(1, inputRecord);
+    int idx = opValueEvaluated(2, inputRecord);
+    // the index is 1 based.
+    idx -= 1;
+    int length = replaceStr.length();
+    if (operands.size() == 4) {
+      length = opValueEvaluated(3, inputRecord);
+    }
+
+    StringBuilder result = new StringBuilder(
+        str.length() + replaceStr.length() - length);
+    result.append(str.substring(0, idx))
+        .append(replaceStr)
+        .append(str.substring(idx + length));
+
+    return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, result.toString());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java
new file mode 100644
index 000000000000..a5e8400d04e2
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * String position operator.
+ *
+ * <p>
+ *   example:
+ *     POSITION(string1 IN string2)
+ *     POSITION(string1 IN string2 FROM integer)
+ * </p>
+ */
+public class BeamSqlPositionExpression extends BeamSqlExpression {
+  public BeamSqlPositionExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.INTEGER);
+  }
+
+  @Override public boolean accept() {
+    if (operands.size() < 2 || operands.size() > 3) {
+      return false;
+    }
+
+    if (!SqlTypeName.CHAR_TYPES.contains(opType(0))
+        || !SqlTypeName.CHAR_TYPES.contains(opType(1))) {
+      return false;
+    }
+
+    if (operands.size() == 3
+        && !SqlTypeName.INT_TYPES.contains(opType(2))) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String targetStr = opValueEvaluated(0, inputRecord);
+    String containingStr = opValueEvaluated(1, inputRecord);
+    int from = -1;
+    if (operands.size() == 3) {
+      Number tmp = opValueEvaluated(2, inputRecord);
+      from = tmp.intValue();
+    }
+
+    int idx = containingStr.indexOf(targetStr, from);
+
+    return BeamSqlPrimitive.of(SqlTypeName.INTEGER, idx);
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpression.java
new file mode 100644
index 000000000000..d931db9f2fa8
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpression.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Base class for all string unary operators.
+ */
+public abstract class BeamSqlStringUnaryExpression extends BeamSqlExpression {
+  public BeamSqlStringUnaryExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  @Override public boolean accept() {
+    if (operands.size() != 1) {
+      return false;
+    }
+
+    if (!SqlTypeName.CHAR_TYPES.contains(opType(0))) {
+      return false;
+    }
+
+    return true;
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpression.java
new file mode 100644
index 000000000000..554a3fc68799
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpression.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'SUBSTRING' operator.
+ *
+ * <p>
+ *   SUBSTRING(string FROM integer)
+ *   SUBSTRING(string FROM integer FOR integer)
+ * </p>
+ */
+public class BeamSqlSubstringExpression extends BeamSqlExpression {
+  public BeamSqlSubstringExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public boolean accept() {
+    if (operands.size() < 2 || operands.size() > 3) {
+      return false;
+    }
+
+    if (!SqlTypeName.CHAR_TYPES.contains(opType(0))
+        || !SqlTypeName.INT_TYPES.contains(opType(1))) {
+      return false;
+    }
+
+    if (operands.size() == 3 && !SqlTypeName.INT_TYPES.contains(opType(2))) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String str = opValueEvaluated(0, inputRecord);
+    int idx = opValueEvaluated(1, inputRecord);
+    int startIdx = idx;
+    if (startIdx > 0) {
+      // NOTE: SQL substring is 1 based(rather than 0 based)
+      startIdx -= 1;
+    } else if (startIdx < 0) {
+      // NOTE: SQL also support negative index...
+      startIdx += str.length();
+    } else {
+      return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "");
+    }
+
+    if (operands.size() == 3) {
+      int length = opValueEvaluated(2, inputRecord);
+      if (length < 0) {
+        length = 0;
+      }
+      int endIdx = Math.min(startIdx + length, str.length());
+      return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.substring(startIdx, endIdx));
+    } else {
+      return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.substring(startIdx));
+    }
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpression.java
new file mode 100644
index 000000000000..d6cad7402dea
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpression.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Trim operator.
+ *
+ * <p>
+ * TRIM( { BOTH | LEADING | TRAILING } string1 FROM string2)
+ * </p>
+ */
+public class BeamSqlTrimExpression extends BeamSqlExpression {
+  public BeamSqlTrimExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public boolean accept() {
+    if (operands.size() != 1 && operands.size() != 3) {
+      return false;
+    }
+
+    if (operands.size() == 1 && !SqlTypeName.CHAR_TYPES.contains(opType(0))) {
+      return false;
+    }
+
+    if (operands.size() == 3
+        && (
+        !SqlTypeName.CHAR_TYPES.contains(opType(0))
+            || !SqlTypeName.CHAR_TYPES.contains(opType(1))
+            || !SqlTypeName.CHAR_TYPES.contains(opType(2)))
+        ) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    if (operands.size() == 1) {
+      return BeamSqlPrimitive.of(SqlTypeName.VARCHAR,
+          opValueEvaluated(0, inputRecord).toString().trim());
+    } else {
+      String type = opValueEvaluated(0, inputRecord);
+      String targetStr = opValueEvaluated(1, inputRecord);
+      String containingStr = opValueEvaluated(2, inputRecord);
+
+      switch (type) {
+        case "LEADING":
+          return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, leadingTrim(containingStr, targetStr));
+        case "TRAILING":
+          return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, trailingTrim(containingStr, targetStr));
+        case "BOTH":
+        default:
+          return BeamSqlPrimitive.of(SqlTypeName.VARCHAR,
+              trailingTrim(leadingTrim(containingStr, targetStr), targetStr));
+      }
+    }
+  }
+
+  static String leadingTrim(String containingStr, String targetStr) {
+    int idx = 0;
+    while (containingStr.startsWith(targetStr, idx)) {
+      idx += targetStr.length();
+    }
+
+    return containingStr.substring(idx);
+  }
+
+  static String trailingTrim(String containingStr, String targetStr) {
+    int idx = containingStr.length() - targetStr.length();
+    while (containingStr.startsWith(targetStr, idx)) {
+      idx -= targetStr.length();
+    }
+
+    idx += targetStr.length();
+    return containingStr.substring(0, idx);
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpression.java
new file mode 100644
index 000000000000..d58a2834f9f7
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'UPPER' operator.
+ */
+public class BeamSqlUpperExpression extends BeamSqlStringUnaryExpression {
+  public BeamSqlUpperExpression(List<BeamSqlExpression> operands) {
+    super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    String str = opValueEvaluated(0, inputRecord);
+    return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, str.toUpperCase());
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/package-info.java
new file mode 100644
index 000000000000..f2c63f399a87
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * String operators.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator.string;
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
index 8df0865395b1..d7379fce8d52 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
@@ -33,6 +33,15 @@
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlModExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlMultiplyExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlPlusExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlCharLengthExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlConcatExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlInitCapExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlLowerExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlOverlayExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlPositionExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlSubstringExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlTrimExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.string.BeamSqlUpperExpression;
 import org.apache.beam.dsls.sql.rel.BeamFilterRel;
 import org.apache.beam.dsls.sql.rel.BeamProjectRel;
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
@@ -121,14 +130,128 @@ private void testBuildArithmeticExpression(SqlOperator fn,
       Class<? extends BeamSqlExpression> clazz) {
     RexNode rexNode;
     BeamSqlExpression exp;
-    rexNode = rexBuilder.makeCall(fn,
+    rexNode = rexBuilder.makeCall(fn, Arrays.asList(
+        rexBuilder.makeBigintLiteral(new BigDecimal(1L)),
+        rexBuilder.makeBigintLiteral(new BigDecimal(1L))
+    ));
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+
+    assertTrue(exp.getClass().equals(clazz));
+  }
+
+  public void testBuildExpression_string()  {
+    RexNode rexNode;
+    BeamSqlExpression exp;
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CONCAT,
         Arrays.asList(
-            rexBuilder.makeBigintLiteral(new BigDecimal(1L)),
-            rexBuilder.makeBigintLiteral(new BigDecimal(1L))
+            rexBuilder.makeLiteral("hello "),
+            rexBuilder.makeLiteral("world")
         )
     );
     exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlConcatExpression);
 
-    assertTrue(exp.getClass().equals(clazz));
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.POSITION,
+        Arrays.asList(
+            rexBuilder.makeLiteral("hello"),
+            rexBuilder.makeLiteral("worldhello")
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlPositionExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.POSITION,
+        Arrays.asList(
+            rexBuilder.makeLiteral("hello"),
+            rexBuilder.makeLiteral("worldhello"),
+            rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlPositionExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CHAR_LENGTH,
+        Arrays.asList(
+            rexBuilder.makeLiteral("hello")
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlCharLengthExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.UPPER,
+        Arrays.asList(
+            rexBuilder.makeLiteral("hello")
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlUpperExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.LOWER,
+        Arrays.asList(
+            rexBuilder.makeLiteral("HELLO")
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlLowerExpression);
+
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.INITCAP,
+        Arrays.asList(
+            rexBuilder.makeLiteral("hello")
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlInitCapExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.TRIM,
+        Arrays.asList(
+            rexBuilder.makeLiteral("BOTH"),
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeLiteral("HELLO")
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlTrimExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.SUBSTRING,
+        Arrays.asList(
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlSubstringExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.SUBSTRING,
+        Arrays.asList(
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeBigintLiteral(BigDecimal.ZERO),
+            rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlSubstringExpression);
+
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.OVERLAY,
+        Arrays.asList(
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlOverlayExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.OVERLAY,
+        Arrays.asList(
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeBigintLiteral(BigDecimal.ZERO),
+            rexBuilder.makeBigintLiteral(BigDecimal.ZERO)
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlOverlayExpression);
   }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java
new file mode 100644
index 000000000000..cd02fdf837de
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlCharLengthExpressionTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlCharLengthExpression.
+ */
+public class BeamSqlCharLengthExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    assertEquals(5,
+        new BeamSqlCharLengthExpression(operands).evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpressionTest.java
new file mode 100644
index 000000000000..ca71dec929e5
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpressionTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlConcatExpression.
+ */
+public class BeamSqlConcatExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertTrue(new BeamSqlConcatExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertFalse(new BeamSqlConcatExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    assertFalse(new BeamSqlConcatExpression(operands).accept());
+  }
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " world"));
+    assertEquals("hello world",
+        new BeamSqlConcatExpression(operands).evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpressionTest.java
new file mode 100644
index 000000000000..b38b033b9bbe
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpressionTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test of BeamSqlInitCapExpression.
+ */
+public class BeamSqlInitCapExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello world"));
+    assertEquals("Hello World",
+        new BeamSqlInitCapExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hEllO wOrld"));
+    assertEquals("Hello World",
+        new BeamSqlInitCapExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello     world"));
+    assertEquals("Hello     World",
+        new BeamSqlInitCapExpression(operands).evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpressionTest.java
new file mode 100644
index 000000000000..fead9dc74f8b
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlLowerExpressionTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test of BeamSqlLowerExpression.
+ */
+public class BeamSqlLowerExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "HELLO"));
+    assertEquals("hello",
+        new BeamSqlLowerExpression(operands).evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
new file mode 100644
index 000000000000..3c4bca525f3a
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlOverlayExpression.
+ */
+public class BeamSqlOverlayExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertTrue(new BeamSqlOverlayExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    assertTrue(new BeamSqlOverlayExpression(operands).accept());
+  }
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+    assertEquals("w3resou3rce",
+        new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
+    assertEquals("w3resou33rce",
+        new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+    assertEquals("w3resou3rce",
+        new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w3333333rce"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 7));
+    assertEquals("w3resouce",
+        new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpressionTest.java
new file mode 100644
index 000000000000..7339466f27f3
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpressionTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlPositionExpression.
+ */
+public class BeamSqlPositionExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+    assertTrue(new BeamSqlPositionExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertTrue(new BeamSqlPositionExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+    assertFalse(new BeamSqlPositionExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertFalse(new BeamSqlPositionExpression(operands).accept());
+  }
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+    assertEquals(5, new BeamSqlPositionExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertEquals(5, new BeamSqlPositionExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertEquals(-1, new BeamSqlPositionExpression(operands).evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java
new file mode 100644
index 000000000000..9bb553f2609b
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlStringUnaryExpressionTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlStringUnaryExpression.
+ */
+public class BeamSqlStringUnaryExpressionTest {
+
+  @Test public void accept() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    assertTrue(new BeamSqlCharLengthExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertFalse(new BeamSqlCharLengthExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    assertFalse(new BeamSqlCharLengthExpression(operands).accept());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
new file mode 100644
index 000000000000..78b2731a346e
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlSubstringExpression.
+ */
+public class BeamSqlSubstringExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertTrue(new BeamSqlSubstringExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    assertTrue(new BeamSqlSubstringExpression(operands).accept());
+  }
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    assertEquals("hello",
+        new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+    assertEquals("he",
+        new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+    assertEquals("hello",
+        new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 100));
+    assertEquals("hello",
+        new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 0));
+    assertEquals("",
+        new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, -1));
+    assertEquals("",
+        new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, -1));
+    assertEquals("o",
+        new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java
new file mode 100644
index 000000000000..8ad33c9f21fd
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlTrimExpression.
+ */
+public class BeamSqlTrimExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " hello "));
+    assertTrue(new BeamSqlTrimExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "LEADING"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+    assertTrue(new BeamSqlTrimExpression(operands).accept());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+    assertFalse(new BeamSqlTrimExpression(operands).accept());
+  }
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "LEADING"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+    assertEquals("__hehe",
+        new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "TRAILING"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+    assertEquals("hehe__",
+        new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "BOTH"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "__"));
+    assertEquals("__",
+        new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " hello "));
+    assertEquals("hello",
+        new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+  }
+
+  @Test public void leadingTrim() throws Exception {
+    assertEquals("__hehe",
+        BeamSqlTrimExpression.leadingTrim("hehe__hehe", "he"));
+  }
+
+  @Test public void trailingTrim() throws Exception {
+    assertEquals("hehe__",
+        BeamSqlTrimExpression.trailingTrim("hehe__hehe", "he"));
+  }
+
+  @Test public void trim() throws Exception {
+    assertEquals("__",
+        BeamSqlTrimExpression.leadingTrim(
+        BeamSqlTrimExpression.trailingTrim("hehe__hehe", "he"), "he"
+        ));
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java
new file mode 100644
index 000000000000..e6f3500f0179
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test of BeamSqlUpperExpression.
+ */
+public class BeamSqlUpperExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    assertEquals("HELLO",
+        new BeamSqlUpperExpression(operands).evaluate(record).getValue());
+  }
+
+}

From 0409c1713eb86bd9e2d2afd22f9b4053473c0485 Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Thu, 4 May 2017 11:44:14 +0800
Subject: [PATCH 373/387] [BEAM-2079] Support TextIO as SQL source/sink

---
 dsls/sql/pom.xml                              |   5 +
 .../sql/schema/text/BeamTextCSVTable.java     |  66 +++++++
 .../schema/text/BeamTextCSVTableIOReader.java | 123 ++++++++++++
 .../schema/text/BeamTextCSVTableIOWriter.java |  76 ++++++++
 .../dsls/sql/schema/text/BeamTextTable.java   |  45 +++++
 .../dsls/sql/schema/text/package-info.java    |  22 +++
 .../sql/schema/text/BeamTextCSVTableTest.java | 179 ++++++++++++++++++
 7 files changed, 516 insertions(+)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextTable.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/package-info.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java

diff --git a/dsls/sql/pom.xml b/dsls/sql/pom.xml
index 6139adafb4ed..15692e9a559c 100644
--- a/dsls/sql/pom.xml
+++ b/dsls/sql/pom.xml
@@ -204,5 +204,10 @@
       <artifactId>hamcrest-all</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-csv</artifactId>
+      <version>1.4</version>
+    </dependency>
   </dependencies>
 </project>
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
new file mode 100644
index 000000000000..b9e6b816529f
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema.text;
+
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.commons.csv.CSVFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@code BeamTextCSVTable} is a {@code BeamTextTable} which formatted in CSV.
+ *
+ * <p>
+ * {@link CSVFormat} itself has many dialects, check its javadoc for more info.
+ * </p>
+ */
+public class BeamTextCSVTable extends BeamTextTable {
+  private static final Logger LOG = LoggerFactory
+      .getLogger(BeamTextCSVTable.class);
+
+  private CSVFormat csvFormat;
+
+  /**
+   * CSV table with {@link CSVFormat#DEFAULT DEFAULT} format.
+   */
+  public BeamTextCSVTable(RelProtoDataType protoDataType, String filePattern)  {
+    this(protoDataType, filePattern, CSVFormat.DEFAULT);
+  }
+
+  public BeamTextCSVTable(RelProtoDataType protoDataType, String filePattern,
+      CSVFormat csvFormat) {
+    super(protoDataType, filePattern);
+    this.csvFormat = csvFormat;
+  }
+
+  @Override
+  public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
+    return new BeamTextCSVTableIOReader(beamSqlRecordType, filePattern, csvFormat);
+  }
+
+  @Override
+  public PTransform<? super PCollection<BeamSQLRow>, PDone> buildIOWriter() {
+    return new BeamTextCSVTableIOWriter(beamSqlRecordType, filePattern, csvFormat);
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
new file mode 100644
index 000000000000..cf7c095f7dfd
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema.text;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringReader;
+
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * IOReader for {@code BeamTextCSVTable}.
+ */
+public class BeamTextCSVTableIOReader
+    extends PTransform<PBegin, PCollection<BeamSQLRow>>
+    implements Serializable {
+  private static final Logger LOG = LoggerFactory.getLogger(BeamTextCSVTableIOReader.class);
+  private String filePattern;
+  protected BeamSQLRecordType beamSqlRecordType;
+  protected CSVFormat csvFormat;
+
+  public BeamTextCSVTableIOReader(BeamSQLRecordType beamSqlRecordType, String filePattern,
+      CSVFormat csvFormat) {
+    this.filePattern = filePattern;
+    this.beamSqlRecordType = beamSqlRecordType;
+    this.csvFormat = csvFormat;
+  }
+
+  @Override
+  public PCollection<BeamSQLRow> expand(PBegin input) {
+    return input.apply("decodeRecord", TextIO.Read.from(filePattern))
+        .apply(ParDo.of(new DoFn<String, BeamSQLRow>() {
+          @ProcessElement
+          public void processElement(ProcessContext ctx) {
+            String str = ctx.element();
+
+            try (StringReader reader = new StringReader(str)) {
+              CSVRecord rawRecord = null;
+              try {
+                CSVParser parser = csvFormat.parse(reader);
+                rawRecord = parser.getRecords().get(0);
+              } catch (IOException e) {
+                throw new IllegalArgumentException("Invalid text filePattern: " + filePattern, e);
+              }
+
+              BeamSQLRow row = new BeamSQLRow(beamSqlRecordType);
+              if (rawRecord.size() != beamSqlRecordType.size()) {
+                throw new IllegalArgumentException(String.format(
+                    "Invalid filePattern: {}, expect %d fields, but actually %d", str,
+                    filePattern, beamSqlRecordType.size(), rawRecord.size()
+                ));
+              } else {
+                for (int idx = 0; idx < beamSqlRecordType.size(); idx++) {
+                  String raw = rawRecord.get(idx);
+                  addFieldWithAutoTypeCasting(row, idx, raw);
+                }
+                ctx.output(row);
+              }
+            }
+          }
+        }));
+  }
+
+  public void addFieldWithAutoTypeCasting(BeamSQLRow row, int idx, String raw) {
+    SqlTypeName columnType = row.getDataType().getFieldsType().get(idx);
+    switch (columnType) {
+      case TINYINT:
+        row.addField(idx, Byte.valueOf(raw));
+        break;
+      case SMALLINT:
+        row.addField(idx, Short.valueOf(raw));
+        break;
+      case INTEGER:
+        row.addField(idx, Integer.valueOf(raw));
+        break;
+      case BIGINT:
+        row.addField(idx, Long.valueOf(raw));
+        break;
+      case FLOAT:
+        row.addField(idx, Float.valueOf(raw));
+        break;
+      case DOUBLE:
+        row.addField(idx, Double.valueOf(raw));
+        break;
+      case VARCHAR:
+        row.addField(idx, raw);
+        break;
+      default:
+        throw new BeamSqlUnsupportedException(String.format(
+            "Column type %s is not supported yet!", columnType));
+    }
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
new file mode 100644
index 000000000000..6104cd8e9491
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema.text;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringWriter;
+
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * IOWriter for {@code BeamTextCSVTable}.
+ */
+public class BeamTextCSVTableIOWriter extends PTransform<PCollection<BeamSQLRow>, PDone>
+    implements Serializable {
+  private static final Logger LOG = LoggerFactory.getLogger(BeamTextCSVTableIOWriter.class);
+
+  private String filePattern;
+  protected BeamSQLRecordType beamSqlRecordType;
+  protected CSVFormat csvFormat;
+
+  public BeamTextCSVTableIOWriter(BeamSQLRecordType beamSqlRecordType, String filePattern,
+      CSVFormat csvFormat) {
+    this.filePattern = filePattern;
+    this.beamSqlRecordType = beamSqlRecordType;
+    this.csvFormat = csvFormat;
+  }
+
+  @Override public PDone expand(PCollection<BeamSQLRow> input) {
+    return input.apply("encodeRecord", ParDo.of(new DoFn<BeamSQLRow, String>() {
+
+      @ProcessElement public void processElement(ProcessContext ctx) {
+        BeamSQLRow row = ctx.element();
+        StringWriter writer = new StringWriter();
+
+        try (CSVPrinter printer = csvFormat.print(writer)) {
+          for (int i = 0; i < row.size(); i++) {
+            printer.print(row.getFieldValue(i).toString());
+          }
+          printer.println();
+        } catch (IOException e) {
+          throw new IllegalArgumentException("Invalid filePattern: " + filePattern, e);
+        }
+
+        ctx.output(writer.toString());
+      }
+    })).apply(TextIO.Write.to(filePattern));
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextTable.java
new file mode 100644
index 000000000000..335376194cac
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextTable.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema.text;
+
+import java.io.Serializable;
+
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamIOType;
+import org.apache.calcite.rel.type.RelProtoDataType;
+
+/**
+ * {@code BeamTextTable} represents a text file/directory(backed by {@code TextIO}).
+ */
+public abstract class BeamTextTable extends BaseBeamTable implements Serializable {
+  protected String filePattern;
+  protected BeamTextTable(RelProtoDataType protoRowType) {
+    super(protoRowType);
+  }
+
+  protected BeamTextTable(RelProtoDataType protoDataType, String filePattern) {
+    super(protoDataType);
+    this.filePattern = filePattern;
+  }
+
+  @Override
+  public BeamIOType getSourceType() {
+    return BeamIOType.BOUNDED;
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/package-info.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/package-info.java
new file mode 100644
index 000000000000..f48f2fe5c250
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Table schema for text files.
+ */
+package org.apache.beam.dsls.sql.schema.text;
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
new file mode 100644
index 000000000000..e06f8da4f8a9
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema.text;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Tests for {@code BeamTextCSVTable}.
+ */
+public class BeamTextCSVTableTest {
+
+  @Rule public TestPipeline pipeline = TestPipeline.create();
+  @Rule public TestPipeline pipeline2 = TestPipeline.create();
+
+  /**
+   * testData.
+   *
+   * <p>
+   * The types of the csv fields are:
+   *     integer,bigint,float,double,string
+   * </p>
+   */
+  private static Object[] data1 = new Object[] { 1, 1L, 1.1F, 1.1, "james" };
+  private static Object[] data2 = new Object[] { 2, 2L, 2.2F, 2.2, "bond" };
+
+  private static List<Object[]> testData = Arrays.asList(data1, data2);
+  private static List<BeamSQLRow> testDataRows = new ArrayList<BeamSQLRow>() {{
+    for (Object[] data : testData) {
+      add(buildRow(data));
+    }
+  }};
+  private static ConcurrentLinkedQueue<Object[]> actualData = new ConcurrentLinkedQueue<>();
+
+  private static Path tempFolder;
+  private static File readerSourceFile;
+  private static File writerTargetFile;
+
+  @Test public void testBuildIOReader() {
+    PCollection<BeamSQLRow> rows = pipeline.apply(
+        new BeamTextCSVTable(buildRowType(), readerSourceFile.getAbsolutePath()).buildIOReader());
+    PAssert.that(rows).containsInAnyOrder(testDataRows);
+    pipeline.run();
+  }
+
+  @Test public void testBuildIOWriter() {
+    // reader from a source file, then write into a target file
+    pipeline.apply(
+        new BeamTextCSVTable(buildRowType(), readerSourceFile.getAbsolutePath()).buildIOReader())
+        .apply(new BeamTextCSVTable(buildRowType(), writerTargetFile.getAbsolutePath())
+            .buildIOWriter());
+    pipeline.run();
+
+    PCollection<BeamSQLRow> rows = pipeline2.apply(
+        new BeamTextCSVTable(buildRowType(), writerTargetFile.getAbsolutePath()).buildIOReader());
+
+    // confirm the two reads match
+    PAssert.that(rows).containsInAnyOrder(testDataRows);
+    pipeline2.run();
+  }
+
+  @BeforeClass public static void setUp() throws IOException {
+    tempFolder = Files.createTempDirectory("BeamTextTableTest");
+    readerSourceFile = writeToFile(testData, "readerSourceFile.txt");
+    writerTargetFile = writeToFile(testData, "writerTargetFile.txt");
+  }
+
+  @AfterClass public static void teardownClass() throws IOException {
+    Files.walkFileTree(tempFolder, new SimpleFileVisitor<Path>() {
+
+      @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+          throws IOException {
+        Files.delete(file);
+        return FileVisitResult.CONTINUE;
+      }
+
+      @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+          throws IOException {
+        Files.delete(dir);
+        return FileVisitResult.CONTINUE;
+      }
+    });
+  }
+
+  private static File writeToFile(List<Object[]> rows, String filename) throws IOException {
+    File file = tempFolder.resolve(filename).toFile();
+    OutputStream output = new FileOutputStream(file);
+    writeToStreamAndClose(rows, output);
+    return file;
+  }
+
+  /**
+   * Helper that writes the given lines (adding a newline in between) to a stream, then closes the
+   * stream.
+   */
+  private static void writeToStreamAndClose(List<Object[]> rows, OutputStream outputStream) {
+    try (PrintStream writer = new PrintStream(outputStream)) {
+      CSVPrinter printer = CSVFormat.DEFAULT.print(writer);
+      for (Object[] row : rows) {
+        for (Object field : row) {
+          printer.print(field);
+        }
+        printer.println();
+      }
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
+  private RelProtoDataType buildRowType() {
+    return new RelProtoDataType() {
+
+      @Override public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder().add("id", SqlTypeName.INTEGER).add("order_id", SqlTypeName.BIGINT)
+            .add("price", SqlTypeName.FLOAT).add("amount", SqlTypeName.DOUBLE)
+            .add("user_name", SqlTypeName.VARCHAR).build();
+      }
+    };
+  }
+
+  private static RelDataType buildRelDataType() {
+    return BeamQueryPlanner.TYPE_FACTORY.builder().add("id", SqlTypeName.INTEGER)
+        .add("order_id", SqlTypeName.BIGINT).add("price", SqlTypeName.FLOAT)
+        .add("amount", SqlTypeName.DOUBLE).add("user_name", SqlTypeName.VARCHAR).build();
+  }
+
+  private static BeamSQLRecordType buildBeamSQLRecordType() {
+    return BeamSQLRecordType.from(buildRelDataType());
+  }
+
+  private static BeamSQLRow buildRow(Object[] data) {
+    return new BeamSQLRow(buildBeamSQLRecordType(), Arrays.asList(data));
+  }
+}

From 8d0cdfeba31930dd726c0da237ae889e0f1eb98c Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Tue, 9 May 2017 23:04:17 -0700
Subject: [PATCH 374/387] [BEAM-2006] window support Add support for
 aggregation: global, HOP, TUMBLE, SESSION, only aggregation function COUNT

fix typo
---
 dsls/sql/pom.xml                              |   6 +-
 .../sql/interpreter/BeamSQLFnExecutor.java    |  20 ++-
 .../operator/BeamSqlPrimitive.java            |  10 ++
 .../operator/BeamSqlWindowEndExpression.java  |  42 +++++
 .../operator/BeamSqlWindowExpression.java     |  50 ++++++
 .../BeamSqlWindowStartExpression.java         |  43 +++++
 .../dsls/sql/planner/BeamPipelineCreator.java |  17 +-
 .../beam/dsls/sql/planner/BeamRuleSets.java   |   4 +-
 .../beam/dsls/sql/rel/BeamAggregationRel.java | 154 +++++++++++++++++
 .../beam/dsls/sql/rel/BeamFilterRel.java      |   4 +-
 .../beam/dsls/sql/rel/BeamIOSinkRel.java      |   2 +-
 .../beam/dsls/sql/rel/BeamIOSourceRel.java    |   2 +-
 .../beam/dsls/sql/rel/BeamProjectRel.java     |   4 +-
 .../dsls/sql/rule/BeamAggregationRule.java    | 163 ++++++++++++++++++
 .../dsls/sql/schema/BeamSQLRecordType.java    |   5 +
 .../beam/dsls/sql/schema/BeamSQLRow.java      |  64 ++++++-
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java |  21 ++-
 .../transform/BeamAggregationTransform.java   | 120 +++++++++++++
 .../dsls/sql/transform/BeamSQLProjectFn.java  |   8 +-
 .../beam/dsls/sql/planner/BasePlanner.java    |  61 ++++---
 .../sql/planner/BeamGroupByExplainTest.java   |  93 ++++++++++
 .../sql/planner/BeamGroupByPipelineTest.java  |  94 ++++++++++
 .../sql/planner/BeamInvalidGroupByTest.java   |  44 +++++
 .../BeamPlannerAggregationSubmitTest.java     | 136 +++++++++++++++
 .../sql/planner/BeamPlannerExplainTest.java   |   6 +-
 .../sql/planner/BeamPlannerSubmitTest.java    |  12 +-
 .../dsls/sql/planner/MockedBeamSQLTable.java  |  38 ++--
 27 files changed, 1137 insertions(+), 86 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowEndExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowStartExpression.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamAggregationRule.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java

diff --git a/dsls/sql/pom.xml b/dsls/sql/pom.xml
index 15692e9a559c..bc658e6969ee 100644
--- a/dsls/sql/pom.xml
+++ b/dsls/sql/pom.xml
@@ -34,7 +34,7 @@
   <properties>
     <timestamp>${maven.build.timestamp}</timestamp>
     <maven.build.timestamp.format>yyyy-MM-dd HH:mm</maven.build.timestamp.format>
-    <calcite-version>1.11.0</calcite-version>
+    <calcite-version>1.12.0</calcite-version>
   </properties>
 
   <build>
@@ -209,5 +209,9 @@
       <artifactId>commons-csv</artifactId>
       <version>1.4</version>
     </dependency>
+    <dependency>
+      <groupId>joda-time</groupId>
+      <artifactId>joda-time</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
index a14d347d27e0..4ae7b337c655 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -34,6 +34,9 @@
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlNotEqualExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlOrExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlWindowEndExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlWindowExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlWindowStartExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlDivideExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlMinusExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.arithmetic.BeamSqlModExpression;
@@ -159,8 +162,21 @@ static BeamSqlExpression buildExpression(RexNode rexNode) {
 
         case "IS NULL":
           return new BeamSqlIsNullExpression(subExps.get(0));
-        case "IS NOT NULL":
-          return new BeamSqlIsNotNullExpression(subExps.get(0));
+      case "IS NOT NULL":
+        return new BeamSqlIsNotNullExpression(subExps.get(0));
+
+      case "HOP":
+      case "TUMBLE":
+      case "SESSION":
+        return new BeamSqlWindowExpression(subExps, node.type.getSqlTypeName());
+      case "HOP_START":
+      case "TUMBLE_START":
+      case "SESSION_START":
+        return new BeamSqlWindowStartExpression();
+      case "HOP_END":
+      case "TUMBLE_END":
+      case "SESSION_END":
+        return new BeamSqlWindowEndExpression();
       default:
         throw new BeamSqlUnsupportedException("Operator: " + opName + " not supported yet!");
       }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
index dbe6c3c0bf98..33095779dea4 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
@@ -18,6 +18,8 @@
 package org.apache.beam.dsls.sql.interpreter.operator;
 
 import java.math.BigDecimal;
+import java.util.Date;
+import java.util.GregorianCalendar;
 import java.util.List;
 import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
@@ -89,6 +91,14 @@ public boolean accept() {
     case CHAR:
     case VARCHAR:
       return value instanceof String || value instanceof NlsString;
+    case TIME:
+      return value instanceof GregorianCalendar;
+    case TIMESTAMP:
+      return value instanceof Date;
+    case INTERVAL_HOUR:
+      return value instanceof BigDecimal;
+    case INTERVAL_MINUTE:
+      return value instanceof BigDecimal;
     default:
       throw new BeamSqlUnsupportedException(outputType.name());
     }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowEndExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowEndExpression.java
new file mode 100644
index 000000000000..96ad81f472d0
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowEndExpression.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.Date;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for {@code HOP_END}, {@code TUMBLE_END}, {@code SESSION_END} operation.
+ *
+ * <p>These operators returns the <em>end</em> timestamp of window.
+ */
+public class BeamSqlWindowEndExpression extends BeamSqlExpression {
+
+  @Override
+  public boolean accept() {
+    return true;
+  }
+
+  @Override
+  public BeamSqlPrimitive<Date> evaluate(BeamSQLRow inputRecord) {
+    return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP,
+        new Date(inputRecord.getWindowEnd().getMillis()));
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowExpression.java
new file mode 100644
index 000000000000..2fb9a48ba5fc
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowExpression.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.Date;
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for {@code HOP}, {@code TUMBLE}, {@code SESSION} operation.
+ *
+ * <p>These functions don't change the timestamp field, instead it's used to indicate
+ * the event_timestamp field, and how the window is defined.
+ */
+public class BeamSqlWindowExpression extends BeamSqlExpression {
+
+  public BeamSqlWindowExpression(List<BeamSqlExpression> operands, SqlTypeName outputType) {
+    super(operands, outputType);
+  }
+
+  @Override
+  public boolean accept() {
+    return operands.get(0).getOutputType().equals(SqlTypeName.DATE)
+        || operands.get(0).getOutputType().equals(SqlTypeName.TIME)
+        || operands.get(0).getOutputType().equals(SqlTypeName.TIMESTAMP);
+  }
+
+  @Override
+  public BeamSqlPrimitive<Date> evaluate(BeamSQLRow inputRecord) {
+    return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP,
+        (Date) operands.get(0).evaluate(inputRecord).getValue());
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowStartExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowStartExpression.java
new file mode 100644
index 000000000000..d0ac260fd1ec
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlWindowStartExpression.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.Date;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for {@code HOP_START}, {@code TUMBLE_START},
+ * {@code SESSION_START} operation.
+ *
+ * <p>These operators returns the <em>start</em> timestamp of window.
+ */
+public class BeamSqlWindowStartExpression extends BeamSqlExpression {
+
+  @Override
+  public boolean accept() {
+    return true;
+  }
+
+  @Override
+  public BeamSqlPrimitive<Date> evaluate(BeamSQLRow inputRecord) {
+    return BeamSqlPrimitive.of(SqlTypeName.TIMESTAMP,
+        new Date(inputRecord.getWindowStart().getMillis()));
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
index 00274a2f2feb..1d7cfd1b5704 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
@@ -19,6 +19,9 @@
 
 import java.util.Map;
 
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
@@ -38,7 +41,7 @@
  */
 public class BeamPipelineCreator {
   private Map<String, BaseBeamTable> sourceTables;
-  private PCollection<BeamSQLRow> latestStream;
+  private Queue<PCollection<BeamSQLRow>> upStreamQueue;
 
   private PipelineOptions options;
 
@@ -53,18 +56,20 @@ public BeamPipelineCreator(Map<String, BaseBeamTable> sourceTables) {
         .as(PipelineOptions.class); // FlinkPipelineOptions.class
     options.setJobName("BeamPlanCreator");
 
+    upStreamQueue = new ConcurrentLinkedQueue<>();
+
     pipeline = Pipeline.create(options);
     CoderRegistry cr = pipeline.getCoderRegistry();
     cr.registerCoder(BeamSQLRow.class, BeamSqlRowCoder.of());
     cr.registerCoder(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
   }
 
-  public PCollection<BeamSQLRow> getLatestStream() {
-    return latestStream;
+  public PCollection<BeamSQLRow> popUpstream() {
+    return upStreamQueue.poll();
   }
 
-  public void setLatestStream(PCollection<BeamSQLRow> latestStream) {
-    this.latestStream = latestStream;
+  public void pushUpstream(PCollection<BeamSQLRow> upstream) {
+    this.upStreamQueue.add(upstream);
   }
 
   public Map<String, BaseBeamTable> getSourceTables() {
@@ -75,7 +80,7 @@ public Pipeline getPipeline() {
     return pipeline;
   }
 
-  public boolean isHasPersistent() {
+  public boolean hasPersistent() {
     return hasPersistent;
   }
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
index 2af31dcbe6f5..acbd43f47ef0 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
@@ -22,6 +22,7 @@
 import java.util.Iterator;
 
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.rule.BeamAggregationRule;
 import org.apache.beam.dsls.sql.rule.BeamFilterRule;
 import org.apache.beam.dsls.sql.rule.BeamIOSinkRule;
 import org.apache.beam.dsls.sql.rule.BeamIOSourceRule;
@@ -38,7 +39,8 @@
 public class BeamRuleSets {
   private static final ImmutableSet<RelOptRule> calciteToBeamConversionRules = ImmutableSet
       .<RelOptRule>builder().add(BeamIOSourceRule.INSTANCE, BeamProjectRule.INSTANCE,
-          BeamFilterRule.INSTANCE, BeamIOSinkRule.INSTANCE)
+          BeamFilterRule.INSTANCE, BeamIOSinkRule.INSTANCE,
+          BeamAggregationRule.INSTANCE)
       .build();
 
   public static RuleSet[] getRuleSets() {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
new file mode 100644
index 000000000000..2c7626d0096a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.rel;
+
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.transform.BeamAggregationTransform;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.WithTimestamps;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Util;
+import org.joda.time.Duration;
+
+/**
+ * {@link BeamRelNode} to replace a {@link Aggregate} node.
+ *
+ */
+public class BeamAggregationRel extends Aggregate implements BeamRelNode {
+  private int windowFieldIdx = -1;
+  private WindowFn<BeamSQLRow, BoundedWindow> windowFn;
+  private Trigger trigger;
+  private Duration allowedLatence = Duration.ZERO;
+
+  public BeamAggregationRel(RelOptCluster cluster, RelTraitSet traits
+      , RelNode child, boolean indicator,
+      ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls
+      , WindowFn windowFn, Trigger trigger, int windowFieldIdx, Duration allowedLatence) {
+    super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls);
+    this.windowFn = windowFn;
+    this.trigger = trigger;
+    this.windowFieldIdx = windowFieldIdx;
+    this.allowedLatence = allowedLatence;
+  }
+
+  @Override
+  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+    RelNode input = getInput();
+    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
+
+    String stageName = BeamSQLRelUtils.getStageName(this);
+
+    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
+    if (windowFieldIdx != -1) {
+      upstream = upstream.apply("assignEventTimestamp", WithTimestamps
+          .<BeamSQLRow>of(new BeamAggregationTransform.WindowTimestampFn(windowFieldIdx)));
+    }
+
+    PCollection<BeamSQLRow> windowStream = upstream.apply("window",
+        Window.<BeamSQLRow>into(windowFn)
+        .triggering(trigger)
+        .withAllowedLateness(allowedLatence)
+        .accumulatingFiredPanes());
+
+    PCollection<KV<BeamSQLRow, BeamSQLRow>> exGroupByStream = windowStream.apply("exGroupBy",
+        WithKeys
+            .of(new BeamAggregationTransform.AggregationGroupByKeyFn(windowFieldIdx, groupSet)));
+
+    PCollection<KV<BeamSQLRow, Iterable<BeamSQLRow>>> groupedStream = exGroupByStream
+        .apply("groupBy", GroupByKey.<BeamSQLRow, BeamSQLRow>create());
+
+    if (aggCalls.size() > 1) {
+      throw new BeamSqlUnsupportedException("only single aggregation is supported now.");
+    }
+
+    AggregateCall aggCall = aggCalls.get(0);
+    switch (aggCall.getAggregation().getName()) {
+    case "COUNT":
+      PCollection<KV<BeamSQLRow, Long>> aggregatedStream = groupedStream.apply("count",
+          Combine.<BeamSQLRow, BeamSQLRow, Long>groupedValues(Count.combineFn()));
+      PCollection<BeamSQLRow> mergedStream = aggregatedStream.apply("mergeRecord",
+          ParDo.of(new BeamAggregationTransform.MergeAggregationRecord(
+              BeamSQLRecordType.from(getRowType()), aggCall.getName())));
+      planCreator.pushUpstream(mergedStream);
+      break;
+    default:
+      //Only support COUNT now, more are added in BEAM-2008
+      throw new BeamSqlUnsupportedException(
+          String.format("Unsupported aggregation [%s]", aggCall.getAggregation().getName()));
+    }
+
+    return planCreator.getPipeline();
+  }
+
+  @Override
+  public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator
+      , ImmutableBitSet groupSet,
+      List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+    return new BeamAggregationRel(getCluster(), traitSet, input, indicator
+        , groupSet, groupSets, aggCalls, windowFn, trigger, windowFieldIdx, allowedLatence);
+  }
+
+  public void setWindowFn(WindowFn windowFn) {
+    this.windowFn = windowFn;
+  }
+
+  public void setTrigger(Trigger trigger) {
+    this.trigger = trigger;
+  }
+
+  public RelWriter explainTerms(RelWriter pw) {
+    // We skip the "groups" element if it is a singleton of "group".
+    pw.item("group", groupSet)
+        .itemIf("window", windowFn, windowFn != null)
+        .itemIf("trigger", trigger, trigger != null)
+        .itemIf("event_time", windowFieldIdx, windowFieldIdx != -1)
+        .itemIf("groups", groupSets, getGroupType() != Group.SIMPLE)
+        .itemIf("indicator", indicator, indicator)
+        .itemIf("aggs", aggCalls, pw.nest());
+    if (!pw.nest()) {
+      for (Ord<AggregateCall> ord : Ord.zip(aggCalls)) {
+        pw.item(Util.first(ord.e.name, "agg#" + ord.i), ord.e);
+      }
+    }
+    return pw;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
index 477be5a16765..e1c5b3e7ebdc 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
@@ -56,14 +56,14 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
 
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
+    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
 
     BeamSQLExpressionExecutor executor = new BeamSQLFnExecutor(this);
 
     PCollection<BeamSQLRow> projectStream = upstream.apply(stageName,
         ParDo.of(new BeamSQLFilterFn(getRelTypeName(), executor)));
 
-    planCreator.setLatestStream(projectStream);
+    planCreator.pushUpstream(projectStream);
 
     return planCreator.getPipeline();
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
index cad0b3c7445e..f38b9e16c501 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
@@ -60,7 +60,7 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
 
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
+    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
 
     String sourceName = Joiner.on('.').join(getTable().getQualifiedName());
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
index 6b1b6cd8c6fe..35382730881b 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
@@ -52,7 +52,7 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
     PCollection<BeamSQLRow> sourceStream = planCreator.getPipeline().apply(stageName,
         sourceTable.buildIOReader());
 
-    planCreator.setLatestStream(sourceStream);
+    planCreator.pushUpstream(sourceStream);
 
     return planCreator.getPipeline();
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
index 7e27ab3773c3..65f5b2003dba 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
@@ -67,14 +67,14 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
 
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> upstream = planCreator.getLatestStream();
+    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
 
     BeamSQLExpressionExecutor executor = new BeamSQLFnExecutor(this);
 
     PCollection<BeamSQLRow> projectStream = upstream.apply(stageName, ParDo
         .of(new BeamSQLProjectFn(getRelTypeName(), executor, BeamSQLRecordType.from(rowType))));
 
-    planCreator.setLatestStream(projectStream);
+    planCreator.pushUpstream(projectStream);
 
     return planCreator.getPipeline();
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamAggregationRule.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamAggregationRule.java
new file mode 100644
index 000000000000..249d02d7ed5a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamAggregationRule.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.rule;
+
+import com.google.common.collect.ImmutableList;
+import java.util.GregorianCalendar;
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.InvalidFieldException;
+import org.apache.beam.dsls.sql.rel.BeamAggregationRel;
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.joda.time.Duration;
+
+/**
+ * Rule to detect the window/trigger settings.
+ *
+ */
+public class BeamAggregationRule extends RelOptRule {
+  public static final BeamAggregationRule INSTANCE =
+      new BeamAggregationRule(Aggregate.class, Project.class, RelFactories.LOGICAL_BUILDER);
+
+  public BeamAggregationRule(
+      Class<? extends Aggregate> aggregateClass,
+      Class<? extends Project> projectClass,
+      RelBuilderFactory relBuilderFactory) {
+    super(
+        operand(aggregateClass,
+            operand(projectClass, any())),
+        relBuilderFactory, null);
+  }
+
+  public BeamAggregationRule(RelOptRuleOperand operand, String description) {
+    super(operand, description);
+  }
+
+  @Override
+  public void onMatch(RelOptRuleCall call) {
+    final Aggregate aggregate = call.rel(0);
+    final Project project = call.rel(1);
+    updateWindowTrigger(call, aggregate, project);
+  }
+
+  private void updateWindowTrigger(RelOptRuleCall call, Aggregate aggregate,
+      Project project) {
+    ImmutableBitSet groupByFields = aggregate.getGroupSet();
+    List<RexNode> projectMapping = project.getProjects();
+
+    WindowFn windowFn = new GlobalWindows();
+    Trigger triggerFn = Repeatedly.forever(AfterWatermark.pastEndOfWindow());
+    int windowFieldIdx = -1;
+    Duration allowedLatence = Duration.ZERO;
+
+    for (int groupField : groupByFields.asList()) {
+      RexNode projNode = projectMapping.get(groupField);
+      if (projNode instanceof RexCall) {
+        SqlOperator op = ((RexCall) projNode).op;
+        ImmutableList<RexNode> parameters = ((RexCall) projNode).operands;
+        String functionName = op.getName();
+        switch (functionName) {
+        case "TUMBLE":
+          windowFieldIdx = groupField;
+          windowFn = FixedWindows
+              .of(Duration.millis(getWindowParameterAsMillis(parameters.get(1))));
+          if (parameters.size() == 3) {
+            GregorianCalendar delayTime = (GregorianCalendar) ((RexLiteral) parameters.get(2))
+                .getValue();
+            triggerFn = createTriggerWithDelay(delayTime);
+            allowedLatence = (Duration.millis(delayTime.getTimeInMillis()));
+          }
+          break;
+        case "HOP":
+          windowFieldIdx = groupField;
+          windowFn = SlidingWindows
+              .of(Duration.millis(getWindowParameterAsMillis(parameters.get(1))))
+              .every(Duration.millis(getWindowParameterAsMillis(parameters.get(2))));
+          if (parameters.size() == 4) {
+            GregorianCalendar delayTime = (GregorianCalendar) ((RexLiteral) parameters.get(3))
+                .getValue();
+            triggerFn = createTriggerWithDelay(delayTime);
+            allowedLatence = (Duration.millis(delayTime.getTimeInMillis()));
+          }
+          break;
+        case "SESSION":
+          windowFieldIdx = groupField;
+          windowFn = Sessions
+              .withGapDuration(Duration.millis(getWindowParameterAsMillis(parameters.get(1))));
+          if (parameters.size() == 3) {
+            GregorianCalendar delayTime = (GregorianCalendar) ((RexLiteral) parameters.get(2))
+                .getValue();
+            triggerFn = createTriggerWithDelay(delayTime);
+            allowedLatence = (Duration.millis(delayTime.getTimeInMillis()));
+          }
+          break;
+        default:
+          break;
+        }
+      }
+    }
+
+    BeamAggregationRel newAggregator = new BeamAggregationRel(aggregate.getCluster(),
+        aggregate.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+        convert(aggregate.getInput(),
+            aggregate.getInput().getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+        aggregate.indicator,
+        aggregate.getGroupSet(),
+        aggregate.getGroupSets(),
+        aggregate.getAggCallList(),
+        windowFn,
+        triggerFn,
+        windowFieldIdx,
+        allowedLatence);
+    call.transformTo(newAggregator);
+  }
+
+  private Trigger createTriggerWithDelay(GregorianCalendar delayTime) {
+    return Repeatedly.forever(AfterWatermark.pastEndOfWindow().withLateFirings(AfterProcessingTime
+        .pastFirstElementInPane().plusDelayOf(Duration.millis(delayTime.getTimeInMillis()))));
+  }
+
+  private long getWindowParameterAsMillis(RexNode parameterNode) {
+    if (parameterNode instanceof RexLiteral) {
+      return RexLiteral.intValue(parameterNode);
+    } else {
+      throw new InvalidFieldException(String.format("[%s] is not valid.", parameterNode));
+    }
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
index e4013bca6720..94531f01aee0 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordType.java
@@ -42,6 +42,11 @@ public static BeamSQLRecordType from(RelDataType tableInfo) {
     return record;
   }
 
+  public void addField(String fieldName, SqlTypeName fieldType) {
+    fieldsName.add(fieldName);
+    fieldsType.add(fieldType);
+  }
+
   public int size() {
     return fieldsName.size();
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
index f9dab8a41920..65f4a41f2142 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
@@ -20,9 +20,14 @@
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.GregorianCalendar;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import org.apache.beam.dsls.sql.exception.InvalidFieldException;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.joda.time.Instant;
 
 /**
  * Repersent a generic ROW record in Beam SQL.
@@ -34,6 +39,9 @@ public class BeamSQLRow implements Serializable {
   private List<Object> dataValues;
   private BeamSQLRecordType dataType;
 
+  private Instant windowStart = new Instant(TimeUnit.MICROSECONDS.toMillis(Long.MIN_VALUE));
+  private Instant windowEnd = new Instant(TimeUnit.MICROSECONDS.toMillis(Long.MAX_VALUE));
+
   public BeamSQLRow(BeamSQLRecordType dataType) {
     this.dataType = dataType;
     this.dataValues = new ArrayList<>();
@@ -50,6 +58,17 @@ public BeamSQLRow(BeamSQLRecordType dataType, List<Object> dataValues) {
     }
   }
 
+  public void updateWindowRange(BeamSQLRow upstreamRecord, BoundedWindow window){
+    windowStart = upstreamRecord.windowStart;
+    windowEnd = upstreamRecord.windowEnd;
+
+    if (window instanceof IntervalWindow) {
+      IntervalWindow iWindow = (IntervalWindow) window;
+      windowStart = iWindow.start();
+      windowEnd = iWindow.end();
+    }
+  }
+
   public void addField(String fieldName, Object fieldValue) {
     addField(dataType.getFieldsName().indexOf(fieldName), fieldValue);
   }
@@ -107,6 +126,18 @@ public void addField(int index, Object fieldValue) {
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
       }
       break;
+    case TIME:
+      if (!(fieldValue instanceof GregorianCalendar)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
+    case TIMESTAMP:
+      if (!(fieldValue instanceof Date)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      }
+      break;
     default:
       throw new UnsupportedDataTypeException(fieldType);
     }
@@ -203,6 +234,20 @@ public Object getFieldValue(int fieldIdx) {
       } else {
         return fieldValue;
       }
+    case TIME:
+      if (!(fieldValue instanceof GregorianCalendar)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
+        return fieldValue;
+      }
+    case TIMESTAMP:
+      if (!(fieldValue instanceof Date)) {
+        throw new InvalidFieldException(
+            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+      } else {
+        return fieldValue;
+      }
     default:
       throw new UnsupportedDataTypeException(fieldType);
     }
@@ -236,9 +281,26 @@ public List<Integer> getNullFields() {
     return nullFields;
   }
 
+  public Instant getWindowStart() {
+    return windowStart;
+  }
+
+  public Instant getWindowEnd() {
+    return windowEnd;
+  }
+
+  public void setWindowStart(Instant windowStart) {
+    this.windowStart = windowStart;
+  }
+
+  public void setWindowEnd(Instant windowEnd) {
+    this.windowEnd = windowEnd;
+  }
+
   @Override
   public String toString() {
-    return "BeamSQLRow [dataValues=" + dataValues + ", dataType=" + dataType + "]";
+    return "BeamSQLRow [nullFields=" + nullFields + ", dataValues=" + dataValues + ", dataType="
+        + dataType + ", windowStart=" + windowStart + ", windowEnd=" + windowEnd + "]";
   }
 
   /**
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index 9b2474a334f1..3100ba55102b 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -20,12 +20,14 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Date;
 import java.util.List;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
 import org.apache.beam.sdk.coders.DoubleCoder;
+import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.StandardCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -43,6 +45,7 @@ public class BeamSqlRowCoder extends StandardCoder<BeamSQLRow>{
   private static final BigEndianIntegerCoder intCoder = BigEndianIntegerCoder.of();
   private static final BigEndianLongCoder longCoder = BigEndianLongCoder.of();
   private static final DoubleCoder doubleCoder = DoubleCoder.of();
+  private static final InstantCoder instantCoder = InstantCoder.of();
 
   private static final BeamSqlRowCoder INSTANCE = new BeamSqlRowCoder();
   private BeamSqlRowCoder(){}
@@ -74,8 +77,7 @@ public void encode(BeamSQLRow value, OutputStream outStream,
           doubleCoder.encode(value.getDouble(idx), outStream, context.nested());
           break;
         case FLOAT:
-          doubleCoder.encode(Double.parseDouble(
-              String.valueOf(value.getFloat(idx))), outStream, context.nested());
+          doubleCoder.encode((double) value.getFloat(idx), outStream, context.nested());
           break;
         case BIGINT:
           longCoder.encode(value.getLong(idx), outStream, context.nested());
@@ -83,13 +85,17 @@ public void encode(BeamSQLRow value, OutputStream outStream,
         case VARCHAR:
           stringCoder.encode(value.getString(idx), outStream, context.nested());
           break;
+        case TIMESTAMP:
+          longCoder.encode(value.getDate(idx).getTime(), outStream, context);
+          break;
 
         default:
           throw new UnsupportedDataTypeException(value.getDataType().getFieldsType().get(idx));
       }
     }
-    //add a dummy field to indicate the end of record
-    intCoder.encode(value.size(), outStream, context);
+
+    instantCoder.encode(value.getWindowStart(), outStream, context.nested());
+    instantCoder.encode(value.getWindowEnd(), outStream, context);
   }
 
   @Override
@@ -128,12 +134,17 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
         case VARCHAR:
           record.addField(idx, stringCoder.decode(inStream, context.nested()));
           break;
+        case TIMESTAMP:
+          record.addField(idx, new Date(longCoder.decode(inStream, context)));
+          break;
 
         default:
           throw new UnsupportedDataTypeException(type.getFieldsType().get(idx));
       }
     }
-    intCoder.decode(inStream, context);
+
+    record.setWindowStart(instantCoder.decode(inStream, context.nested()));
+    record.setWindowEnd(instantCoder.decode(inStream, context));
 
     return record;
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java
new file mode 100644
index 000000000000..f478363c657f
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.transform;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.joda.time.Instant;
+
+/**
+ * Collections of {@code PTransform} and {@code DoFn} used to perform GROUP-BY operation.
+ */
+public class BeamAggregationTransform implements Serializable{
+  /**
+   * Merge KV to single record.
+   */
+  public static class MergeAggregationRecord extends DoFn<KV<BeamSQLRow, Long>, BeamSQLRow> {
+    private BeamSQLRecordType outRecordType;
+    private String aggFieldName;
+
+    public MergeAggregationRecord(BeamSQLRecordType outRecordType, String aggFieldName) {
+      this.outRecordType = outRecordType;
+      this.aggFieldName = aggFieldName;
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext c, BoundedWindow window) {
+      BeamSQLRow outRecord = new BeamSQLRow(outRecordType);
+      outRecord.updateWindowRange(c.element().getKey(), window);
+
+      KV<BeamSQLRow, Long> kvRecord = c.element();
+      for (String f : kvRecord.getKey().getDataType().getFieldsName()) {
+        outRecord.addField(f, kvRecord.getKey().getFieldValue(f));
+      }
+      outRecord.addField(aggFieldName, kvRecord.getValue());
+
+//      if (c.pane().isLast()) {
+      c.output(outRecord);
+//      }
+    }
+  }
+
+  /**
+   * extract group-by fields.
+   */
+  public static class AggregationGroupByKeyFn
+      implements SerializableFunction<BeamSQLRow, BeamSQLRow> {
+    private List<Integer> groupByKeys;
+
+    public AggregationGroupByKeyFn(int windowFieldIdx, ImmutableBitSet groupSet) {
+      this.groupByKeys = new ArrayList<>();
+      for (int i : groupSet.asList()) {
+        if (i != windowFieldIdx) {
+          groupByKeys.add(i);
+        }
+      }
+    }
+
+    @Override
+    public BeamSQLRow apply(BeamSQLRow input) {
+      BeamSQLRecordType typeOfKey = exTypeOfKeyRecord(input.getDataType());
+      BeamSQLRow keyOfRecord = new BeamSQLRow(typeOfKey);
+      keyOfRecord.updateWindowRange(input, null);
+
+      for (int idx = 0; idx < groupByKeys.size(); ++idx) {
+        keyOfRecord.addField(idx, input.getFieldValue(groupByKeys.get(idx)));
+      }
+      return keyOfRecord;
+    }
+
+    private BeamSQLRecordType exTypeOfKeyRecord(BeamSQLRecordType dataType) {
+      BeamSQLRecordType typeOfKey = new BeamSQLRecordType();
+      for (int idx : groupByKeys) {
+        typeOfKey.addField(dataType.getFieldsName().get(idx), dataType.getFieldsType().get(idx));
+      }
+      return typeOfKey;
+    }
+
+  }
+
+  /**
+   * Assign event timestamp.
+   */
+  public static class WindowTimestampFn implements SerializableFunction<BeamSQLRow, Instant> {
+    private int windowFieldIdx = -1;
+
+    public WindowTimestampFn(int windowFieldIdx) {
+      super();
+      this.windowFieldIdx = windowFieldIdx;
+    }
+
+    @Override
+    public Instant apply(BeamSQLRow input) {
+      return new Instant(input.getDate(windowFieldIdx).getTime());
+    }
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
index d0180571c24e..79dd67f74e92 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
@@ -23,6 +23,7 @@
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 
 /**
  *
@@ -48,10 +49,13 @@ public void setup() {
   }
 
   @ProcessElement
-  public void processElement(ProcessContext c) {
-    List<Object> results = executor.execute(c.element());
+  public void processElement(ProcessContext c, BoundedWindow window) {
+    BeamSQLRow inputRecord = c.element();
+    List<Object> results = executor.execute(inputRecord);
 
     BeamSQLRow outRow = new BeamSQLRow(outputRecordType);
+    outRow.updateWindowRange(inputRecord, window);
+
     for (int idx = 0; idx < results.size(); ++idx) {
       outRow.addField(idx, results.get(idx));
     }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
index 625fb719233a..055a4d4e8e6c 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
@@ -18,17 +18,18 @@
 package org.apache.beam.dsls.sql.planner;
 
 import java.util.Arrays;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
-
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelProtoDataType;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 /**
@@ -36,50 +37,60 @@
  *
  */
 public class BasePlanner {
-  public static BeamSqlRunner runner;
+  public static BeamSqlRunner runner = new BeamSqlRunner();
 
   @BeforeClass
   public static void prepare() {
-    runner = new BeamSqlRunner();
-
     runner.addTable("ORDER_DETAILS", getTable());
     runner.addTable("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
     runner.addTable("SUB_ORDER_RAM", getTable());
   }
 
-  @AfterClass
-  public static void close(){
-    runner = null;
-  }
-
   private static BaseBeamTable getTable() {
     final RelProtoDataType protoRowType = new RelProtoDataType() {
       @Override
       public RelDataType apply(RelDataTypeFactory a0) {
-        return a0.builder()
-            .add("order_id", SqlTypeName.BIGINT)
-            .add("site_id", SqlTypeName.INTEGER)
-            .add("price", SqlTypeName.DOUBLE)
-            .add("shipping", SqlTypeName.FLOAT)
-            .add("notes", SqlTypeName.VARCHAR)
-            .build();
+        return a0.builder().add("order_id", SqlTypeName.BIGINT).add("site_id", SqlTypeName.INTEGER)
+            .add("price", SqlTypeName.DOUBLE).add("order_time", SqlTypeName.TIMESTAMP).build();
       }
     };
 
-    return new MockedBeamSQLTable(protoRowType);
+    BeamSQLRecordType dataType = BeamSQLRecordType.from(
+        protoRowType.apply(BeamQueryPlanner.TYPE_FACTORY));
+    BeamSQLRow row1 = new BeamSQLRow(dataType);
+    row1.addField(0, 12345L);
+    row1.addField(1, 0);
+    row1.addField(2, 10.5);
+    row1.addField(3, new Date());
+
+    BeamSQLRow row2 = new BeamSQLRow(dataType);
+    row2.addField(0, 12345L);
+    row2.addField(1, 1);
+    row2.addField(2, 20.5);
+    row2.addField(3, new Date());
+
+    BeamSQLRow row3 = new BeamSQLRow(dataType);
+    row3.addField(0, 12345L);
+    row3.addField(1, 0);
+    row3.addField(2, 20.5);
+    row3.addField(3, new Date());
+
+    BeamSQLRow row4 = new BeamSQLRow(dataType);
+    row4.addField(0, null);
+    row4.addField(1, null);
+    row4.addField(2, 20.5);
+    row4.addField(3, new Date());
+
+    return new MockedBeamSQLTable(protoRowType).withInputRecords(
+        Arrays.asList(row1, row2, row3, row4));
   }
 
   public static BaseBeamTable getTable(String bootstrapServer, String topic) {
     final RelProtoDataType protoRowType = new RelProtoDataType() {
       @Override
       public RelDataType apply(RelDataTypeFactory a0) {
-        return a0.builder()
-            .add("order_id", SqlTypeName.BIGINT)
-            .add("site_id", SqlTypeName.INTEGER)
-            .add("price", SqlTypeName.DOUBLE)
-            .add("shipping", SqlTypeName.FLOAT)
-            .add("notes", SqlTypeName.VARCHAR)
-            .build();
+        return a0.builder().add("order_id", SqlTypeName.BIGINT).add("site_id", SqlTypeName.INTEGER)
+            .add("price", SqlTypeName.DOUBLE).add("order_time", SqlTypeName.TIMESTAMP).build();
       }
     };
 
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
new file mode 100644
index 000000000000..566c5748b7da
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.planner;
+
+import org.junit.Test;
+
+/**
+ * Test group-by methods.
+ *
+ */
+public class BeamGroupByExplainTest extends BasePlanner {
+
+  /**
+   * GROUP-BY without window operation, and grouped fields.
+   */
+  @Test
+  public void testSimpleGroupExplain() throws Exception {
+    String sql = "SELECT COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 ";
+    String plan = runner.explainQuery(sql);
+  }
+
+  /**
+   * GROUP-BY without window operation, and grouped fields.
+   */
+  @Test
+  public void testSimpleGroup2Explain() throws Exception {
+    String sql = "SELECT site_id" + ", COUNT(*) " + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY site_id";
+    String plan = runner.explainQuery(sql);
+  }
+
+  /**
+   * GROUP-BY with TUMBLE window.
+   */
+  @Test
+  public void testTumbleExplain() throws Exception {
+    String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
+        + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
+    String plan = runner.explainQuery(sql);
+  }
+
+  /**
+   * GROUP-BY with TUMBLE window.
+   */
+  @Test
+  public void testTumbleWithDelayExplain() throws Exception {
+    String sql = "SELECT order_id, site_id, "
+        + "TUMBLE_START(order_time, INTERVAL '1' HOUR, TIME '00:00:01')"
+        + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 "
+        + "GROUP BY order_id, site_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR, TIME '00:00:01')";
+    String plan = runner.explainQuery(sql);
+  }
+
+  /**
+   * GROUP-BY with HOP window.
+   */
+  @Test
+  public void testHopExplain() throws Exception {
+    String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
+        + ", HOP(order_time, INTERVAL '5' MINUTE, INTERVAL '1' HOUR)";
+    String plan = runner.explainQuery(sql);
+  }
+
+  /**
+   * GROUP-BY with SESSION window.
+   */
+  @Test
+  public void testSessionExplain() throws Exception {
+    String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
+        + ", SESSION(order_time, INTERVAL '5' MINUTE)";
+    String plan = runner.explainQuery(sql);
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
new file mode 100644
index 000000000000..d5f81250833d
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.planner;
+
+import org.apache.beam.sdk.Pipeline;
+import org.junit.Test;
+
+/**
+ * Test group-by methods.
+ *
+ */
+public class BeamGroupByPipelineTest extends BasePlanner {
+
+  /**
+   * GROUP-BY without window operation, and grouped fields.
+   */
+  @Test
+  public void testSimpleGroupExplain() throws Exception {
+    String sql = "SELECT COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 ";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+  }
+
+  /**
+   * GROUP-BY without window operation, and grouped fields.
+   */
+  @Test
+  public void testSimpleGroup2Explain() throws Exception {
+    String sql = "SELECT site_id" + ", COUNT(*) " + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY site_id";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+  }
+
+  /**
+   * GROUP-BY with TUMBLE window.
+   */
+  @Test
+  public void testTumbleExplain() throws Exception {
+    String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
+        + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+  }
+
+  /**
+   * GROUP-BY with TUMBLE window.
+   */
+  @Test
+  public void testTumbleWithDelayExplain() throws Exception {
+    String sql = "SELECT order_id, site_id, "
+        + "TUMBLE_START(order_time, INTERVAL '1' HOUR, TIME '00:00:01')"
+        + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 "
+        + "GROUP BY order_id, site_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR, TIME '00:00:01')";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+  }
+
+  /**
+   * GROUP-BY with HOP window.
+   */
+  @Test
+  public void testHopExplain() throws Exception {
+    String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
+        + ", HOP(order_time, INTERVAL '5' MINUTE, INTERVAL '1' HOUR)";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+  }
+
+  /**
+   * GROUP-BY with SESSION window.
+   */
+  @Test
+  public void testSessionExplain() throws Exception {
+    String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
+        + ", SESSION(order_time, INTERVAL '5' MINUTE)";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java
new file mode 100644
index 000000000000..83ab8716f6f8
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.planner;
+
+import org.apache.calcite.tools.ValidationException;
+import org.junit.Test;
+
+/**
+ * Test group-by methods.
+ *
+ */
+public class BeamInvalidGroupByTest extends BasePlanner {
+
+  @Test(expected = ValidationException.class)
+  public void testTumble2Explain() throws Exception {
+    String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY order_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
+    String plan = runner.explainQuery(sql);
+  }
+
+  @Test(expected = ValidationException.class)
+  public void testTumble3Explain() throws Exception {
+    String sql = "SELECT order_id, site_id, TUMBLE(order_time, INTERVAL '1' HOUR)"
+        + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 "
+        + "GROUP BY order_id, site_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
+    String plan = runner.explainQuery(sql);
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
new file mode 100644
index 000000000000..39a56143b27d
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.planner;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests to execute a query.
+ *
+ */
+public class BeamPlannerAggregationSubmitTest {
+  public static DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+  public static BeamSqlRunner runner = new BeamSqlRunner();
+
+  @BeforeClass
+  public static void prepare() throws ParseException {
+    runner.addTable("ORDER_DETAILS", getOrderTable());
+    runner.addTable("ORDER_SUMMARY", getSummaryTable());
+  }
+
+  private static BaseBeamTable getOrderTable() throws ParseException {
+    final RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder().add("order_id", SqlTypeName.BIGINT)
+            .add("site_id", SqlTypeName.INTEGER)
+            .add("order_time", SqlTypeName.TIMESTAMP).build();
+      }
+    };
+
+    BeamSQLRecordType dataType = BeamSQLRecordType.from(
+        protoRowType.apply(BeamQueryPlanner.TYPE_FACTORY));
+    BeamSQLRow row1 = new BeamSQLRow(dataType);
+    row1.addField(0, 12345L);
+    row1.addField(1, 1);
+    row1.addField(2, format.parse("2017-01-01 01:02:03"));
+
+    BeamSQLRow row2 = new BeamSQLRow(dataType);
+    row2.addField(0, 12345L);
+    row2.addField(1, 0);
+    row2.addField(2, format.parse("2017-01-01 01:03:04"));
+
+    BeamSQLRow row3 = new BeamSQLRow(dataType);
+    row3.addField(0, 12345L);
+    row3.addField(1, 0);
+    row3.addField(2, format.parse("2017-01-01 02:03:04"));
+
+    BeamSQLRow row4 = new BeamSQLRow(dataType);
+    row4.addField(0, 2132L);
+    row4.addField(1, 0);
+    row4.addField(2, format.parse("2017-01-01 03:04:05"));
+
+    return new MockedBeamSQLTable(protoRowType).withInputRecords(
+        Arrays.asList(row1
+            , row2, row3, row4
+            ));
+
+  }
+
+  private static BaseBeamTable getSummaryTable() {
+    final RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder()
+            .add("site_id", SqlTypeName.INTEGER)
+            .add("agg_hour", SqlTypeName.TIMESTAMP)
+            .add("size", SqlTypeName.BIGINT).build();
+      }
+    };
+    return new MockedBeamSQLTable(protoRowType);
+  }
+
+
+  @Test
+  public void selectWithWindowAggregation() throws Exception{
+    String sql = "INSERT INTO ORDER_SUMMARY(SITE_ID, agg_hour, SIZE) "
+        + "SELECT site_id, TUMBLE_START(order_time, INTERVAL '1' HOUR, TIME '00:00:01')"
+        + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 1 " + "GROUP BY site_id"
+        + ", TUMBLE(order_time, INTERVAL '1' HOUR, TIME '00:00:01')";
+
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+
+    pipeline.run().waitUntilFinish();
+
+    Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
+    BeamSQLRow result = MockedBeamSQLTable.CONTENT.get(0);
+    Assert.assertEquals(1, result.getInteger(0));
+    Assert.assertEquals(format.parse("2017-01-01 01:00:00"), result.getDate(1));
+    Assert.assertEquals(1L, result.getLong(2));
+  }
+
+  @Test
+  public void selectWithoutWindowAggregation() throws Exception{
+    String sql = "INSERT INTO ORDER_SUMMARY(SITE_ID, SIZE) "
+        + "SELECT site_id, COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
+        + "WHERE SITE_ID = 0 " + "GROUP BY site_id";
+
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+
+    pipeline.run().waitUntilFinish();
+
+    Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
+    Assert.assertEquals("site_id=0,agg_hour=null,size=3",
+        MockedBeamSQLTable.CONTENT.get(0).valueInString());
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
index 5d1052b55930..9b6b20a1743c 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
@@ -32,7 +32,7 @@ public void selectAll() throws Exception {
     String plan = runner.explainQuery(sql);
 
     String expectedPlan =
-        "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], shipping=[$3], notes=[$4])\n"
+        "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], order_time=[$3])\n"
         + "  BeamIOSourceRel(table=[[ORDER_DETAILS]])\n";
     Assert.assertEquals("explain doesn't match", expectedPlan, plan);
   }
@@ -58,12 +58,10 @@ public void insertSelectFilter() throws Exception {
 
     String expectedPlan =
         "BeamIOSinkRel(table=[[SUB_ORDER]], operation=[INSERT], flattened=[true])\n"
-        + "  BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], shipping=[null],"
-            + " notes=[null])\n"
+        + "  BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], order_time=[null])\n"
         + "    BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2])\n"
         + "      BeamFilterRel(condition=[AND(=($1, 0), >($2, 20))])\n"
         + "        BeamIOSourceRel(table=[[ORDER_DETAILS]])\n";
     Assert.assertEquals("explain doesn't match", expectedPlan, plan);
   }
-
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
index 1ca9eb33994d..54350491c996 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.dsls.sql.planner;
 
+import org.apache.beam.sdk.Pipeline;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -28,13 +29,14 @@ public class BeamPlannerSubmitTest extends BasePlanner {
   @Test
   public void insertSelectFilter() throws Exception {
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
-        + " order_id, site_id, price " + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 and price > 20";
+        + " order_id, site_id, price "
+        + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 and price > 20";
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
 
-    runner.submitQuery(sql);
+    pipeline.run().waitUntilFinish();
 
     Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
-    Assert.assertEquals("order_id=12345,site_id=0,price=20.5,shipping=null,notes=null",
-        MockedBeamSQLTable.CONTENT.get(0));
+    Assert.assertTrue(MockedBeamSQLTable.CONTENT.get(0).valueInString()
+        .contains("order_id=12345,site_id=0,price=20.5,order_time="));
   }
-
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
index 538607f5d6cb..611bd73a89dd 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
@@ -19,7 +19,6 @@
 
 import java.util.ArrayList;
 import java.util.List;
-
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamIOType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
@@ -38,12 +37,19 @@
  */
 public class MockedBeamSQLTable extends BaseBeamTable {
 
-  public static final List<String> CONTENT = new ArrayList<>();
+  public static final List<BeamSQLRow> CONTENT = new ArrayList<>();
+
+  private List<BeamSQLRow> inputRecords;
 
   public MockedBeamSQLTable(RelProtoDataType protoRowType) {
     super(protoRowType);
   }
 
+  public MockedBeamSQLTable withInputRecords(List<BeamSQLRow> inputRecords){
+    this.inputRecords = inputRecords;
+    return this;
+  }
+
   @Override
   public BeamIOType getSourceType() {
     return BeamIOType.UNBOUNDED;
@@ -51,31 +57,7 @@ public BeamIOType getSourceType() {
 
   @Override
   public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
-    BeamSQLRow row1 = new BeamSQLRow(beamSqlRecordType);
-    row1.addField(0, 12345L);
-    row1.addField(1, 0);
-    row1.addField(2, 10.5);
-    row1.addField(3, 123.4f);
-
-    BeamSQLRow row2 = new BeamSQLRow(beamSqlRecordType);
-    row2.addField(0, 12345L);
-    row2.addField(1, 1);
-    row2.addField(2, 20.5);
-    row2.addField(3, 234.5f);
-
-    BeamSQLRow row3 = new BeamSQLRow(beamSqlRecordType);
-    row3.addField(0, 12345L);
-    row3.addField(1, 0);
-    row3.addField(2, 20.5);
-    row3.addField(3, 345.6f);
-
-    BeamSQLRow row4 = new BeamSQLRow(beamSqlRecordType);
-    row4.addField(0, null);
-    row4.addField(1, null);
-    row4.addField(2, 20.5);
-    row4.addField(3, 456.7f);
-
-    return Create.of(row1, row2, row3);
+    return Create.of(inputRecords);
   }
 
   @Override
@@ -100,7 +82,7 @@ public void setup() {
 
         @ProcessElement
         public void processElement(ProcessContext c) {
-          CONTENT.add(c.element().valueInString());
+          CONTENT.add(c.element());
         }
 
         @Teardown

From 7972290f4ef0649f534cce040b7c270ca97c4c71 Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Wed, 10 May 2017 16:38:13 -0700
Subject: [PATCH 375/387] Support common-used aggregation functions in SQL,
 including:   COUNT,SUM,AVG,MAX,MIN

rename BeamAggregationTransform to BeamAggregationTransforms

update comments
---
 .../operator/BeamSqlPrimitive.java            |  35 +
 .../beam/dsls/sql/rel/BeamAggregationRel.java |  40 +-
 .../beam/dsls/sql/schema/BeamSQLRow.java      |   4 +
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java |   4 +-
 .../transform/BeamAggregationTransform.java   | 120 ----
 .../transform/BeamAggregationTransforms.java  | 671 ++++++++++++++++++
 .../BeamAggregationTransformTest.java         | 436 ++++++++++++
 .../transform/BeamTransformBaseTest.java      |  96 +++
 8 files changed, 1261 insertions(+), 145 deletions(-)
 delete mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransforms.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamTransformBaseTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
index 33095779dea4..a5938f3f71eb 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
@@ -65,6 +65,41 @@ public T getValue() {
     return value;
   }
 
+  public long getLong() {
+    return (Long) getValue();
+  }
+
+  public double getDouble() {
+    return (Double) getValue();
+  }
+
+  public float getFloat() {
+    return (Float) getValue();
+  }
+
+  public int getInteger() {
+    return (Integer) getValue();
+  }
+
+  public short getShort() {
+    return (Short) getValue();
+  }
+
+  public byte getByte() {
+    return (Byte) getValue();
+  }
+  public boolean getBoolean() {
+    return (Boolean) getValue();
+  }
+
+  public String getString() {
+    return (String) getValue();
+  }
+
+  public Date getDate() {
+    return (Date) getValue();
+  }
+
   @Override
   public boolean accept() {
     if (value == null) {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
index 2c7626d0096a..ab98cc470a1c 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
@@ -18,15 +18,13 @@
 package org.apache.beam.dsls.sql.rel;
 
 import java.util.List;
-import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
 import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
-import org.apache.beam.dsls.sql.transform.BeamAggregationTransform;
+import org.apache.beam.dsls.sql.transform.BeamAggregationTransforms;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.WithKeys;
@@ -79,7 +77,7 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
     PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
     if (windowFieldIdx != -1) {
       upstream = upstream.apply("assignEventTimestamp", WithTimestamps
-          .<BeamSQLRow>of(new BeamAggregationTransform.WindowTimestampFn(windowFieldIdx)));
+          .<BeamSQLRow>of(new BeamAggregationTransforms.WindowTimestampFn(windowFieldIdx)));
     }
 
     PCollection<BeamSQLRow> windowStream = upstream.apply("window",
@@ -88,32 +86,26 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
         .withAllowedLateness(allowedLatence)
         .accumulatingFiredPanes());
 
+    //1. extract fields in group-by key part
     PCollection<KV<BeamSQLRow, BeamSQLRow>> exGroupByStream = windowStream.apply("exGroupBy",
         WithKeys
-            .of(new BeamAggregationTransform.AggregationGroupByKeyFn(windowFieldIdx, groupSet)));
+            .of(new BeamAggregationTransforms.AggregationGroupByKeyFn(windowFieldIdx, groupSet)));
 
+    //2. apply a GroupByKey.
     PCollection<KV<BeamSQLRow, Iterable<BeamSQLRow>>> groupedStream = exGroupByStream
         .apply("groupBy", GroupByKey.<BeamSQLRow, BeamSQLRow>create());
 
-    if (aggCalls.size() > 1) {
-      throw new BeamSqlUnsupportedException("only single aggregation is supported now.");
-    }
-
-    AggregateCall aggCall = aggCalls.get(0);
-    switch (aggCall.getAggregation().getName()) {
-    case "COUNT":
-      PCollection<KV<BeamSQLRow, Long>> aggregatedStream = groupedStream.apply("count",
-          Combine.<BeamSQLRow, BeamSQLRow, Long>groupedValues(Count.combineFn()));
-      PCollection<BeamSQLRow> mergedStream = aggregatedStream.apply("mergeRecord",
-          ParDo.of(new BeamAggregationTransform.MergeAggregationRecord(
-              BeamSQLRecordType.from(getRowType()), aggCall.getName())));
-      planCreator.pushUpstream(mergedStream);
-      break;
-    default:
-      //Only support COUNT now, more are added in BEAM-2008
-      throw new BeamSqlUnsupportedException(
-          String.format("Unsupported aggregation [%s]", aggCall.getAggregation().getName()));
-    }
+    //3. run aggregation functions
+    PCollection<KV<BeamSQLRow, BeamSQLRow>> aggregatedStream = groupedStream.apply("aggregation",
+        Combine.<BeamSQLRow, BeamSQLRow, BeamSQLRow>groupedValues(
+            new BeamAggregationTransforms.AggregationCombineFn(getAggCallList(),
+                BeamSQLRecordType.from(input.getRowType()))));
+
+    //4. flat KV to a single record
+    PCollection<BeamSQLRow> mergedStream = aggregatedStream.apply("mergeRecord",
+        ParDo.of(new BeamAggregationTransforms.MergeAggregationRecord(
+            BeamSQLRecordType.from(getRowType()), getAggCallList())));
+    planCreator.pushUpstream(mergedStream);
 
     return planCreator.getPipeline();
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
index 65f4a41f2142..5bdd5d2afed3 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
@@ -144,6 +144,10 @@ public void addField(int index, Object fieldValue) {
     dataValues.set(index, fieldValue);
   }
 
+  public byte getByte(int idx) {
+    return (Byte) getFieldValue(idx);
+  }
+
   public short getShort(int idx) {
     return (Short) getFieldValue(idx);
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index 3100ba55102b..0accb9a4f6a3 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -70,9 +70,11 @@ public void encode(BeamSQLRow value, OutputStream outStream,
           intCoder.encode(value.getInteger(idx), outStream, context.nested());
           break;
         case SMALLINT:
-        case TINYINT:
           intCoder.encode((int) value.getShort(idx), outStream, context.nested());
           break;
+        case TINYINT:
+          intCoder.encode((int) value.getByte(idx), outStream, context.nested());
+          break;
         case DOUBLE:
           doubleCoder.encode(value.getDouble(idx), outStream, context.nested());
           break;
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java
deleted file mode 100644
index f478363c657f..000000000000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransform.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.transform;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
-import org.apache.beam.dsls.sql.schema.BeamSQLRow;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.values.KV;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.joda.time.Instant;
-
-/**
- * Collections of {@code PTransform} and {@code DoFn} used to perform GROUP-BY operation.
- */
-public class BeamAggregationTransform implements Serializable{
-  /**
-   * Merge KV to single record.
-   */
-  public static class MergeAggregationRecord extends DoFn<KV<BeamSQLRow, Long>, BeamSQLRow> {
-    private BeamSQLRecordType outRecordType;
-    private String aggFieldName;
-
-    public MergeAggregationRecord(BeamSQLRecordType outRecordType, String aggFieldName) {
-      this.outRecordType = outRecordType;
-      this.aggFieldName = aggFieldName;
-    }
-
-    @ProcessElement
-    public void processElement(ProcessContext c, BoundedWindow window) {
-      BeamSQLRow outRecord = new BeamSQLRow(outRecordType);
-      outRecord.updateWindowRange(c.element().getKey(), window);
-
-      KV<BeamSQLRow, Long> kvRecord = c.element();
-      for (String f : kvRecord.getKey().getDataType().getFieldsName()) {
-        outRecord.addField(f, kvRecord.getKey().getFieldValue(f));
-      }
-      outRecord.addField(aggFieldName, kvRecord.getValue());
-
-//      if (c.pane().isLast()) {
-      c.output(outRecord);
-//      }
-    }
-  }
-
-  /**
-   * extract group-by fields.
-   */
-  public static class AggregationGroupByKeyFn
-      implements SerializableFunction<BeamSQLRow, BeamSQLRow> {
-    private List<Integer> groupByKeys;
-
-    public AggregationGroupByKeyFn(int windowFieldIdx, ImmutableBitSet groupSet) {
-      this.groupByKeys = new ArrayList<>();
-      for (int i : groupSet.asList()) {
-        if (i != windowFieldIdx) {
-          groupByKeys.add(i);
-        }
-      }
-    }
-
-    @Override
-    public BeamSQLRow apply(BeamSQLRow input) {
-      BeamSQLRecordType typeOfKey = exTypeOfKeyRecord(input.getDataType());
-      BeamSQLRow keyOfRecord = new BeamSQLRow(typeOfKey);
-      keyOfRecord.updateWindowRange(input, null);
-
-      for (int idx = 0; idx < groupByKeys.size(); ++idx) {
-        keyOfRecord.addField(idx, input.getFieldValue(groupByKeys.get(idx)));
-      }
-      return keyOfRecord;
-    }
-
-    private BeamSQLRecordType exTypeOfKeyRecord(BeamSQLRecordType dataType) {
-      BeamSQLRecordType typeOfKey = new BeamSQLRecordType();
-      for (int idx : groupByKeys) {
-        typeOfKey.addField(dataType.getFieldsName().get(idx), dataType.getFieldsType().get(idx));
-      }
-      return typeOfKey;
-    }
-
-  }
-
-  /**
-   * Assign event timestamp.
-   */
-  public static class WindowTimestampFn implements SerializableFunction<BeamSQLRow, Instant> {
-    private int windowFieldIdx = -1;
-
-    public WindowTimestampFn(int windowFieldIdx) {
-      super();
-      this.windowFieldIdx = windowFieldIdx;
-    }
-
-    @Override
-    public Instant apply(BeamSQLRow input) {
-      return new Instant(input.getDate(windowFieldIdx).getTime());
-    }
-  }
-
-}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransforms.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransforms.java
new file mode 100644
index 000000000000..943c8970b6a1
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamAggregationTransforms.java
@@ -0,0 +1,671 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.transform;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlInputRefExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.joda.time.Instant;
+
+/**
+ * Collections of {@code PTransform} and {@code DoFn} used to perform GROUP-BY operation.
+ */
+public class BeamAggregationTransforms implements Serializable{
+  /**
+   * Merge KV to single record.
+   */
+  public static class MergeAggregationRecord extends DoFn<KV<BeamSQLRow, BeamSQLRow>, BeamSQLRow> {
+    private BeamSQLRecordType outRecordType;
+    private List<String> aggFieldNames;
+
+    public MergeAggregationRecord(BeamSQLRecordType outRecordType, List<AggregateCall> aggList) {
+      this.outRecordType = outRecordType;
+      this.aggFieldNames = new ArrayList<>();
+      for (AggregateCall ac : aggList) {
+        aggFieldNames.add(ac.getName());
+      }
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext c, BoundedWindow window) {
+      BeamSQLRow outRecord = new BeamSQLRow(outRecordType);
+      outRecord.updateWindowRange(c.element().getKey(), window);
+
+      KV<BeamSQLRow, BeamSQLRow> kvRecord = c.element();
+      for (String f : kvRecord.getKey().getDataType().getFieldsName()) {
+        outRecord.addField(f, kvRecord.getKey().getFieldValue(f));
+      }
+      for (int idx = 0; idx < aggFieldNames.size(); ++idx) {
+        outRecord.addField(aggFieldNames.get(idx), kvRecord.getValue().getFieldValue(idx));
+      }
+
+      // if (c.pane().isLast()) {
+      c.output(outRecord);
+      // }
+    }
+  }
+
+  /**
+   * extract group-by fields.
+   */
+  public static class AggregationGroupByKeyFn
+      implements SerializableFunction<BeamSQLRow, BeamSQLRow> {
+    private List<Integer> groupByKeys;
+
+    public AggregationGroupByKeyFn(int windowFieldIdx, ImmutableBitSet groupSet) {
+      this.groupByKeys = new ArrayList<>();
+      for (int i : groupSet.asList()) {
+        if (i != windowFieldIdx) {
+          groupByKeys.add(i);
+        }
+      }
+    }
+
+    @Override
+    public BeamSQLRow apply(BeamSQLRow input) {
+      BeamSQLRecordType typeOfKey = exTypeOfKeyRecord(input.getDataType());
+      BeamSQLRow keyOfRecord = new BeamSQLRow(typeOfKey);
+      keyOfRecord.updateWindowRange(input, null);
+
+      for (int idx = 0; idx < groupByKeys.size(); ++idx) {
+        keyOfRecord.addField(idx, input.getFieldValue(groupByKeys.get(idx)));
+      }
+      return keyOfRecord;
+    }
+
+    private BeamSQLRecordType exTypeOfKeyRecord(BeamSQLRecordType dataType) {
+      BeamSQLRecordType typeOfKey = new BeamSQLRecordType();
+      for (int idx : groupByKeys) {
+        typeOfKey.addField(dataType.getFieldsName().get(idx), dataType.getFieldsType().get(idx));
+      }
+      return typeOfKey;
+    }
+
+  }
+
+  /**
+   * Assign event timestamp.
+   */
+  public static class WindowTimestampFn implements SerializableFunction<BeamSQLRow, Instant> {
+    private int windowFieldIdx = -1;
+
+    public WindowTimestampFn(int windowFieldIdx) {
+      super();
+      this.windowFieldIdx = windowFieldIdx;
+    }
+
+    @Override
+    public Instant apply(BeamSQLRow input) {
+      return new Instant(input.getDate(windowFieldIdx).getTime());
+    }
+  }
+
+  /**
+   * Aggregation function which supports COUNT, MAX, MIN, SUM, AVG.
+   *
+   * <p>Multiple aggregation functions are combined together.
+   * For each aggregation function, it may accept part of all data types:<br>
+   *   1). COUNT works for any data type;<br>
+   *   2). MAX/MIN works for INT, LONG, FLOAT, DOUBLE, DECIMAL, SMALLINT, TINYINT, TIMESTAMP;<br>
+   *   3). SUM/AVG works for INT, LONG, FLOAT, DOUBLE, DECIMAL, SMALLINT, TINYINT;<br>
+   *
+   */
+  public static class AggregationCombineFn extends CombineFn<BeamSQLRow, BeamSQLRow, BeamSQLRow> {
+    private BeamSQLRecordType aggDataType;
+
+    private int countIndex = -1;
+
+    List<String> aggFunctions;
+    List<BeamSqlExpression> aggElementExpressions;
+
+    public AggregationCombineFn(List<AggregateCall> aggregationCalls,
+        BeamSQLRecordType sourceRowRecordType) {
+      this.aggDataType = new BeamSQLRecordType();
+      this.aggFunctions = new ArrayList<>();
+      this.aggElementExpressions = new ArrayList<>();
+
+      boolean hasAvg = false;
+      boolean hasCount = false;
+      int countIndex = -1;
+      for (int idx = 0; idx < aggregationCalls.size(); ++idx) {
+        AggregateCall ac = aggregationCalls.get(idx);
+        //verify it's supported.
+        verifySupportedAggregation(ac);
+
+        aggDataType.addField(ac.name, ac.type.getSqlTypeName());
+
+        SqlAggFunction aggFn = ac.getAggregation();
+        switch (aggFn.getName()) {
+        case "COUNT":
+          aggElementExpressions.add(BeamSqlPrimitive.<Long>of(SqlTypeName.BIGINT, 1L));
+          hasCount = true;
+          countIndex = idx;
+          break;
+        case "SUM":
+        case "MAX":
+        case "MIN":
+        case "AVG":
+          int refIndex = ac.getArgList().get(0);
+          aggElementExpressions.add(new BeamSqlInputRefExpression(
+              sourceRowRecordType.getFieldsType().get(refIndex), refIndex));
+          if ("AVG".equals(aggFn.getName())) {
+            hasAvg = true;
+          }
+          break;
+
+        default:
+          break;
+        }
+        aggFunctions.add(aggFn.getName());
+      }
+      // add a COUNT holder if only have AVG
+      if (hasAvg && !hasCount) {
+        aggDataType.addField("__COUNT", SqlTypeName.BIGINT);
+
+        aggFunctions.add("COUNT");
+        aggElementExpressions.add(BeamSqlPrimitive.<Long>of(SqlTypeName.BIGINT, 1L));
+
+        hasCount = true;
+        countIndex = aggDataType.size() - 1;
+      }
+
+      this.countIndex = countIndex;
+    }
+
+    private void verifySupportedAggregation(AggregateCall ac) {
+      //donot support DISTINCT
+      if (ac.isDistinct()) {
+        throw new BeamSqlUnsupportedException("DISTINCT is not supported yet.");
+      }
+      String aggFnName = ac.getAggregation().getName();
+      switch (aggFnName) {
+      case "COUNT":
+        //COUNT works for any data type;
+        break;
+      case "SUM":
+        // SUM only support for INT, LONG, FLOAT, DOUBLE, DECIMAL, SMALLINT,
+        // TINYINT now
+        if (!Arrays
+            .asList(SqlTypeName.INTEGER, SqlTypeName.BIGINT, SqlTypeName.FLOAT, SqlTypeName.DOUBLE,
+                SqlTypeName.SMALLINT, SqlTypeName.TINYINT)
+            .contains(ac.type.getSqlTypeName())) {
+          throw new BeamSqlUnsupportedException(
+              "SUM only support for INT, LONG, FLOAT, DOUBLE, SMALLINT, TINYINT");
+        }
+        break;
+      case "MAX":
+      case "MIN":
+        // MAX/MIN only support for INT, LONG, FLOAT, DOUBLE, DECIMAL, SMALLINT,
+        // TINYINT, TIMESTAMP now
+        if (!Arrays.asList(SqlTypeName.INTEGER, SqlTypeName.BIGINT, SqlTypeName.FLOAT,
+            SqlTypeName.DOUBLE, SqlTypeName.SMALLINT, SqlTypeName.TINYINT,
+            SqlTypeName.TIMESTAMP).contains(ac.type.getSqlTypeName())) {
+          throw new BeamSqlUnsupportedException("MAX/MIN only support for INT, LONG, FLOAT,"
+              + " DOUBLE, SMALLINT, TINYINT, TIMESTAMP");
+        }
+        break;
+      case "AVG":
+        // AVG only support for INT, LONG, FLOAT, DOUBLE, DECIMAL, SMALLINT,
+        // TINYINT now
+        if (!Arrays
+            .asList(SqlTypeName.INTEGER, SqlTypeName.BIGINT, SqlTypeName.FLOAT, SqlTypeName.DOUBLE,
+                SqlTypeName.SMALLINT, SqlTypeName.TINYINT)
+            .contains(ac.type.getSqlTypeName())) {
+          throw new BeamSqlUnsupportedException(
+              "AVG only support for INT, LONG, FLOAT, DOUBLE, SMALLINT, TINYINT");
+        }
+        break;
+      default:
+        throw new BeamSqlUnsupportedException(
+            String.format("[%s] is not supported.", aggFnName));
+      }
+    }
+
+    @Override
+    public BeamSQLRow createAccumulator() {
+      BeamSQLRow initialRecord = new BeamSQLRow(aggDataType);
+      for (int idx = 0; idx < aggElementExpressions.size(); ++idx) {
+        BeamSqlExpression ex = aggElementExpressions.get(idx);
+        String aggFnName = aggFunctions.get(idx);
+        switch (aggFnName) {
+        case "COUNT":
+          initialRecord.addField(idx, 0L);
+          break;
+        case "AVG":
+        case "SUM":
+          //for both AVG/SUM, a summary value is hold at first.
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            initialRecord.addField(idx, 0);
+            break;
+          case BIGINT:
+            initialRecord.addField(idx, 0L);
+            break;
+          case SMALLINT:
+            initialRecord.addField(idx, (short) 0);
+            break;
+          case TINYINT:
+            initialRecord.addField(idx, (byte) 0);
+            break;
+          case FLOAT:
+            initialRecord.addField(idx, 0.0f);
+            break;
+          case DOUBLE:
+            initialRecord.addField(idx, 0.0);
+            break;
+          default:
+            break;
+          }
+          break;
+        case "MAX":
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            initialRecord.addField(idx, Integer.MIN_VALUE);
+            break;
+          case BIGINT:
+            initialRecord.addField(idx, Long.MIN_VALUE);
+            break;
+          case SMALLINT:
+            initialRecord.addField(idx, Short.MIN_VALUE);
+            break;
+          case TINYINT:
+            initialRecord.addField(idx, Byte.MIN_VALUE);
+            break;
+          case FLOAT:
+            initialRecord.addField(idx, Float.MIN_VALUE);
+            break;
+          case DOUBLE:
+            initialRecord.addField(idx, Double.MIN_VALUE);
+            break;
+          case TIMESTAMP:
+            initialRecord.addField(idx, new Date(0));
+            break;
+          default:
+            break;
+          }
+          break;
+        case "MIN":
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            initialRecord.addField(idx, Integer.MAX_VALUE);
+            break;
+          case BIGINT:
+            initialRecord.addField(idx, Long.MAX_VALUE);
+            break;
+          case SMALLINT:
+            initialRecord.addField(idx, Short.MAX_VALUE);
+            break;
+          case TINYINT:
+            initialRecord.addField(idx, Byte.MAX_VALUE);
+            break;
+          case FLOAT:
+            initialRecord.addField(idx, Float.MAX_VALUE);
+            break;
+          case DOUBLE:
+            initialRecord.addField(idx, Double.MAX_VALUE);
+            break;
+          case TIMESTAMP:
+            initialRecord.addField(idx, new Date(Long.MAX_VALUE));
+            break;
+          default:
+            break;
+          }
+          break;
+        default:
+          break;
+        }
+      }
+      return initialRecord;
+    }
+
+    @Override
+    public BeamSQLRow addInput(BeamSQLRow accumulator, BeamSQLRow input) {
+      BeamSQLRow deltaRecord = new BeamSQLRow(aggDataType);
+      for (int idx = 0; idx < aggElementExpressions.size(); ++idx) {
+        BeamSqlExpression ex = aggElementExpressions.get(idx);
+        String aggFnName = aggFunctions.get(idx);
+        switch (aggFnName) {
+        case "COUNT":
+          deltaRecord.addField(idx, 1 + accumulator.getLong(idx));
+          break;
+        case "AVG":
+        case "SUM":
+          // for both AVG/SUM, a summary value is hold at first.
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            deltaRecord.addField(idx,
+                ex.evaluate(input).getInteger() + accumulator.getInteger(idx));
+            break;
+          case BIGINT:
+            deltaRecord.addField(idx, ex.evaluate(input).getLong() + accumulator.getLong(idx));
+            break;
+          case SMALLINT:
+            deltaRecord.addField(idx,
+                (short) (ex.evaluate(input).getShort() + accumulator.getShort(idx)));
+            break;
+          case TINYINT:
+            deltaRecord.addField(idx,
+                (byte) (ex.evaluate(input).getByte() + accumulator.getByte(idx)));
+            break;
+          case FLOAT:
+            deltaRecord.addField(idx,
+                (float) (ex.evaluate(input).getFloat() + accumulator.getFloat(idx)));
+            break;
+          case DOUBLE:
+            deltaRecord.addField(idx, ex.evaluate(input).getDouble() + accumulator.getDouble(idx));
+            break;
+          default:
+            break;
+          }
+          break;
+        case "MAX":
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            deltaRecord.addField(idx,
+                Math.max(ex.evaluate(input).getInteger(), accumulator.getInteger(idx)));
+            break;
+          case BIGINT:
+            deltaRecord.addField(idx,
+                Math.max(ex.evaluate(input).getLong(), accumulator.getLong(idx)));
+            break;
+          case SMALLINT:
+            deltaRecord.addField(idx,
+                (short) Math.max(ex.evaluate(input).getShort(), accumulator.getShort(idx)));
+            break;
+          case TINYINT:
+            deltaRecord.addField(idx,
+                (byte) Math.max(ex.evaluate(input).getByte(), accumulator.getByte(idx)));
+            break;
+          case FLOAT:
+            deltaRecord.addField(idx,
+                Math.max(ex.evaluate(input).getFloat(), accumulator.getFloat(idx)));
+            break;
+          case DOUBLE:
+            deltaRecord.addField(idx,
+                Math.max(ex.evaluate(input).getDouble(), accumulator.getDouble(idx)));
+            break;
+          case TIMESTAMP:
+            Date preDate = accumulator.getDate(idx);
+            Date nowDate = ex.evaluate(input).getDate();
+            deltaRecord.addField(idx, preDate.getTime() > nowDate.getTime() ? preDate : nowDate);
+            break;
+          default:
+            break;
+          }
+          break;
+        case "MIN":
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            deltaRecord.addField(idx,
+                Math.min(ex.evaluate(input).getInteger(), accumulator.getInteger(idx)));
+            break;
+          case BIGINT:
+            deltaRecord.addField(idx,
+                Math.min(ex.evaluate(input).getLong(), accumulator.getLong(idx)));
+            break;
+          case SMALLINT:
+            deltaRecord.addField(idx,
+                (short) Math.min(ex.evaluate(input).getShort(), accumulator.getShort(idx)));
+            break;
+          case TINYINT:
+            deltaRecord.addField(idx,
+                (byte) Math.min(ex.evaluate(input).getByte(), accumulator.getByte(idx)));
+            break;
+          case FLOAT:
+            deltaRecord.addField(idx,
+                Math.min(ex.evaluate(input).getFloat(), accumulator.getFloat(idx)));
+            break;
+          case DOUBLE:
+            deltaRecord.addField(idx,
+                Math.min(ex.evaluate(input).getDouble(), accumulator.getDouble(idx)));
+            break;
+          case TIMESTAMP:
+            Date preDate = accumulator.getDate(idx);
+            Date nowDate = ex.evaluate(input).getDate();
+            deltaRecord.addField(idx, preDate.getTime() < nowDate.getTime() ? preDate : nowDate);
+            break;
+          default:
+            break;
+          }
+          break;
+        default:
+          break;
+        }
+      }
+      return deltaRecord;
+    }
+
+    @Override
+    public BeamSQLRow mergeAccumulators(Iterable<BeamSQLRow> accumulators) {
+      BeamSQLRow deltaRecord = new BeamSQLRow(aggDataType);
+
+      while (accumulators.iterator().hasNext()) {
+        BeamSQLRow sa = accumulators.iterator().next();
+        for (int idx = 0; idx < aggElementExpressions.size(); ++idx) {
+          BeamSqlExpression ex = aggElementExpressions.get(idx);
+          String aggFnName = aggFunctions.get(idx);
+          switch (aggFnName) {
+          case "COUNT":
+            deltaRecord.addField(idx, deltaRecord.getLong(idx) + sa.getLong(idx));
+            break;
+          case "AVG":
+          case "SUM":
+            // for both AVG/SUM, a summary value is hold at first.
+            switch (ex.getOutputType()) {
+            case INTEGER:
+              deltaRecord.addField(idx, deltaRecord.getInteger(idx) + sa.getInteger(idx));
+              break;
+            case BIGINT:
+              deltaRecord.addField(idx, deltaRecord.getLong(idx) + sa.getLong(idx));
+              break;
+            case SMALLINT:
+              deltaRecord.addField(idx, (short) (deltaRecord.getShort(idx) + sa.getShort(idx)));
+              break;
+            case TINYINT:
+              deltaRecord.addField(idx, (byte) (deltaRecord.getByte(idx) + sa.getByte(idx)));
+              break;
+            case FLOAT:
+              deltaRecord.addField(idx, (float) (deltaRecord.getFloat(idx) + sa.getFloat(idx)));
+              break;
+            case DOUBLE:
+              deltaRecord.addField(idx, deltaRecord.getDouble(idx) + sa.getDouble(idx));
+              break;
+            default:
+              break;
+            }
+            break;
+          case "MAX":
+            switch (ex.getOutputType()) {
+            case INTEGER:
+              deltaRecord.addField(idx, Math.max(deltaRecord.getInteger(idx), sa.getInteger(idx)));
+              break;
+            case BIGINT:
+              deltaRecord.addField(idx, Math.max(deltaRecord.getLong(idx), sa.getLong(idx)));
+              break;
+            case SMALLINT:
+              deltaRecord.addField(idx,
+                  (short) Math.max(deltaRecord.getShort(idx), sa.getShort(idx)));
+              break;
+            case TINYINT:
+              deltaRecord.addField(idx, (byte) Math.max(deltaRecord.getByte(idx), sa.getByte(idx)));
+              break;
+            case FLOAT:
+              deltaRecord.addField(idx, Math.max(deltaRecord.getFloat(idx), sa.getFloat(idx)));
+              break;
+            case DOUBLE:
+              deltaRecord.addField(idx, Math.max(deltaRecord.getDouble(idx), sa.getDouble(idx)));
+              break;
+            case TIMESTAMP:
+              Date preDate = deltaRecord.getDate(idx);
+              Date nowDate = sa.getDate(idx);
+              deltaRecord.addField(idx, preDate.getTime() > nowDate.getTime() ? preDate : nowDate);
+              break;
+            default:
+              break;
+            }
+            break;
+          case "MIN":
+            switch (ex.getOutputType()) {
+            case INTEGER:
+              deltaRecord.addField(idx, Math.min(deltaRecord.getInteger(idx), sa.getInteger(idx)));
+              break;
+            case BIGINT:
+              deltaRecord.addField(idx, Math.min(deltaRecord.getLong(idx), sa.getLong(idx)));
+              break;
+            case SMALLINT:
+              deltaRecord.addField(idx,
+                  (short) Math.min(deltaRecord.getShort(idx), sa.getShort(idx)));
+              break;
+            case TINYINT:
+              deltaRecord.addField(idx, (byte) Math.min(deltaRecord.getByte(idx), sa.getByte(idx)));
+              break;
+            case FLOAT:
+              deltaRecord.addField(idx, Math.min(deltaRecord.getFloat(idx), sa.getFloat(idx)));
+              break;
+            case DOUBLE:
+              deltaRecord.addField(idx, Math.min(deltaRecord.getDouble(idx), sa.getDouble(idx)));
+              break;
+            case TIMESTAMP:
+              Date preDate = deltaRecord.getDate(idx);
+              Date nowDate = sa.getDate(idx);
+              deltaRecord.addField(idx, preDate.getTime() < nowDate.getTime() ? preDate : nowDate);
+              break;
+            default:
+              break;
+            }
+            break;
+          default:
+            break;
+          }
+        }
+      }
+      return deltaRecord;
+    }
+
+    @Override
+    public BeamSQLRow extractOutput(BeamSQLRow accumulator) {
+      BeamSQLRow finalRecord = new BeamSQLRow(aggDataType);
+      for (int idx = 0; idx < aggElementExpressions.size(); ++idx) {
+        BeamSqlExpression ex = aggElementExpressions.get(idx);
+        String aggFnName = aggFunctions.get(idx);
+        switch (aggFnName) {
+        case "COUNT":
+          finalRecord.addField(idx, accumulator.getLong(idx));
+          break;
+        case "AVG":
+          long count = accumulator.getLong(countIndex);
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            finalRecord.addField(idx, (int) (accumulator.getInteger(idx) / count));
+            break;
+          case BIGINT:
+            finalRecord.addField(idx, accumulator.getLong(idx) / count);
+            break;
+          case SMALLINT:
+            finalRecord.addField(idx, (short) (accumulator.getShort(idx) / count));
+            break;
+          case TINYINT:
+            finalRecord.addField(idx, (byte) (accumulator.getByte(idx) / count));
+            break;
+          case FLOAT:
+            finalRecord.addField(idx, (float) (accumulator.getFloat(idx) / count));
+            break;
+          case DOUBLE:
+            finalRecord.addField(idx, accumulator.getDouble(idx) / count);
+            break;
+          default:
+            break;
+          }
+          break;
+        case "SUM":
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            finalRecord.addField(idx, accumulator.getInteger(idx));
+            break;
+          case BIGINT:
+            finalRecord.addField(idx, accumulator.getLong(idx));
+            break;
+          case SMALLINT:
+            finalRecord.addField(idx, accumulator.getShort(idx));
+            break;
+          case TINYINT:
+            finalRecord.addField(idx, accumulator.getByte(idx));
+            break;
+          case FLOAT:
+            finalRecord.addField(idx, accumulator.getFloat(idx));
+            break;
+          case DOUBLE:
+            finalRecord.addField(idx, accumulator.getDouble(idx));
+            break;
+          default:
+            break;
+          }
+          break;
+        case "MAX":
+        case "MIN":
+          switch (ex.getOutputType()) {
+          case INTEGER:
+            finalRecord.addField(idx, accumulator.getInteger(idx));
+            break;
+          case BIGINT:
+            finalRecord.addField(idx, accumulator.getLong(idx));
+            break;
+          case SMALLINT:
+            finalRecord.addField(idx, accumulator.getShort(idx));
+            break;
+          case TINYINT:
+            finalRecord.addField(idx, accumulator.getByte(idx));
+            break;
+          case FLOAT:
+            finalRecord.addField(idx, accumulator.getFloat(idx));
+            break;
+          case DOUBLE:
+            finalRecord.addField(idx, accumulator.getDouble(idx));
+            break;
+          case TIMESTAMP:
+            finalRecord.addField(idx, accumulator.getDate(idx));
+            break;
+          default:
+            break;
+          }
+          break;
+        default:
+          break;
+        }
+      }
+      return finalRecord;
+    }
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
new file mode 100644
index 000000000000..f174b9cae67e
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
@@ -0,0 +1,436 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.schema.transform;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordTypeCoder;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
+import org.apache.beam.dsls.sql.transform.BeamAggregationTransforms;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlAvgAggFunction;
+import org.apache.calcite.sql.fun.SqlCountAggFunction;
+import org.apache.calcite.sql.fun.SqlMinMaxAggFunction;
+import org.apache.calcite.sql.fun.SqlSumAggFunction;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link BeamAggregationTransforms}.
+ *
+ */
+public class BeamAggregationTransformTest extends BeamTransformBaseTest{
+
+  @Rule
+  public TestPipeline p = TestPipeline.create();
+
+  private List<AggregateCall> aggCalls;
+  private BeamSQLRecordType keyType = initTypeOfSqlRow(
+      Arrays.asList(KV.of("f_int", SqlTypeName.INTEGER)));
+
+  /**
+   * This step equals to below query.
+   * <pre>
+   * SELECT `f_int`
+   * , COUNT(*) AS `size`
+   * , SUM(`f_long`) AS `sum1`, AVG(`f_long`) AS `avg1`
+   * , MAX(`f_long`) AS `max1`, MIN(`f_long`) AS `min1`
+   * , SUM(`f_short`) AS `sum2`, AVG(`f_short`) AS `avg2`
+   * , MAX(`f_short`) AS `max2`, MIN(`f_short`) AS `min2`
+   * , SUM(`f_byte`) AS `sum3`, AVG(`f_byte`) AS `avg3`
+   * , MAX(`f_byte`) AS `max3`, MIN(`f_byte`) AS `min3`
+   * , SUM(`f_float`) AS `sum4`, AVG(`f_float`) AS `avg4`
+   * , MAX(`f_float`) AS `max4`, MIN(`f_float`) AS `min4`
+   * , SUM(`f_double`) AS `sum5`, AVG(`f_double`) AS `avg5`
+   * , MAX(`f_double`) AS `max5`, MIN(`f_double`) AS `min5`
+   * , MAX(`f_timestamp`) AS `max7`, MIN(`f_timestamp`) AS `min7`
+   * ,SUM(`f_int2`) AS `sum8`, AVG(`f_int2`) AS `avg8`
+   * , MAX(`f_int2`) AS `max8`, MIN(`f_int2`) AS `min8`
+   * FROM TABLE_NAME
+   * GROUP BY `f_int`
+   * </pre>
+   * @throws ParseException
+   */
+  @Test
+  public void testCountPerElementBasic() throws ParseException {
+    setupEnvironment();
+
+    PCollection<BeamSQLRow> input = p.apply(Create.of(inputRows));
+
+    //1. extract fields in group-by key part
+    PCollection<KV<BeamSQLRow, BeamSQLRow>> exGroupByStream = input.apply("exGroupBy",
+        WithKeys
+            .of(new BeamAggregationTransforms.AggregationGroupByKeyFn(-1, ImmutableBitSet.of(0))));
+
+    //2. apply a GroupByKey.
+    PCollection<KV<BeamSQLRow, Iterable<BeamSQLRow>>> groupedStream = exGroupByStream
+        .apply("groupBy", GroupByKey.<BeamSQLRow, BeamSQLRow>create());
+
+    //3. run aggregation functions
+    PCollection<KV<BeamSQLRow, BeamSQLRow>> aggregatedStream = groupedStream.apply("aggregation",
+        Combine.<BeamSQLRow, BeamSQLRow, BeamSQLRow>groupedValues(
+            new BeamAggregationTransforms.AggregationCombineFn(aggCalls, inputRowType)));
+
+    //4. flat KV to a single record
+    PCollection<BeamSQLRow> mergedStream = aggregatedStream.apply("mergeRecord",
+        ParDo.of(new BeamAggregationTransforms.MergeAggregationRecord(
+            BeamSQLRecordType.from(prepareFinalRowType()), aggCalls)));
+
+    //assert function BeamAggregationTransform.AggregationGroupByKeyFn
+    PAssert.that(exGroupByStream).containsInAnyOrder(prepareResultOfAggregationGroupByKeyFn());
+
+    //assert BeamAggregationTransform.AggregationCombineFn
+    PAssert.that(aggregatedStream).containsInAnyOrder(prepareResultOfAggregationCombineFn());
+
+  //assert BeamAggregationTransform.MergeAggregationRecord
+    PAssert.that(mergedStream).containsInAnyOrder(prepareResultOfMergeAggregationRecord());
+
+    p.run();
+}
+
+  private void setupEnvironment() {
+    regiesterCoder();
+    prepareAggregationCalls();
+  }
+
+  /**
+   * Add Coders in BeamSQL.
+   */
+  private void regiesterCoder() {
+    CoderRegistry cr = p.getCoderRegistry();
+    cr.registerCoder(BeamSQLRow.class, BeamSqlRowCoder.of());
+    cr.registerCoder(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
+  }
+
+  /**
+   * create list of all {@link AggregateCall}.
+   */
+  @SuppressWarnings("deprecation")
+  private void prepareAggregationCalls() {
+    //aggregations for all data type
+    aggCalls = new ArrayList<>();
+    aggCalls.add(
+        new AggregateCall(new SqlCountAggFunction(), false,
+            Arrays.<Integer>asList(),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+            "count")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlSumAggFunction(
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT)), false,
+            Arrays.<Integer>asList(1),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+            "sum1")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+            Arrays.<Integer>asList(1),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+            "avg1")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+            Arrays.<Integer>asList(1),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+            "max1")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+            Arrays.<Integer>asList(1),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT),
+            "min1")
+        );
+
+    aggCalls.add(
+        new AggregateCall(new SqlSumAggFunction(
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT)), false,
+            Arrays.<Integer>asList(2),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+            "sum2")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+            Arrays.<Integer>asList(2),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+            "avg2")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+            Arrays.<Integer>asList(2),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+            "max2")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+            Arrays.<Integer>asList(2),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.SMALLINT),
+            "min2")
+        );
+
+    aggCalls.add(
+        new AggregateCall(
+            new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT)),
+            false,
+            Arrays.<Integer>asList(3),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+            "sum3")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+            Arrays.<Integer>asList(3),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+            "avg3")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+            Arrays.<Integer>asList(3),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+            "max3")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+            Arrays.<Integer>asList(3),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TINYINT),
+            "min3")
+        );
+
+    aggCalls.add(
+        new AggregateCall(
+            new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT)),
+            false,
+            Arrays.<Integer>asList(4),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+            "sum4")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+            Arrays.<Integer>asList(4),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+            "avg4")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+            Arrays.<Integer>asList(4),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+            "max4")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+            Arrays.<Integer>asList(4),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.FLOAT),
+            "min4")
+        );
+
+    aggCalls.add(
+        new AggregateCall(
+            new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE)),
+            false,
+            Arrays.<Integer>asList(5),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+            "sum5")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+            Arrays.<Integer>asList(5),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+            "avg5")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+            Arrays.<Integer>asList(5),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+            "max5")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+            Arrays.<Integer>asList(5),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DOUBLE),
+            "min5")
+        );
+
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+            Arrays.<Integer>asList(7),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TIMESTAMP),
+            "max7")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+            Arrays.<Integer>asList(7),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.TIMESTAMP),
+            "min7")
+        );
+
+    aggCalls.add(
+        new AggregateCall(
+            new SqlSumAggFunction(new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER)),
+            false,
+            Arrays.<Integer>asList(8),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+            "sum8")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlAvgAggFunction(SqlKind.AVG), false,
+            Arrays.<Integer>asList(8),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+            "avg8")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MAX), false,
+            Arrays.<Integer>asList(8),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+            "max8")
+        );
+    aggCalls.add(
+        new AggregateCall(new SqlMinMaxAggFunction(SqlKind.MIN), false,
+            Arrays.<Integer>asList(8),
+            new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.INTEGER),
+            "min8")
+        );
+  }
+
+  /**
+   * expected results after {@link BeamAggregationTransforms.AggregationGroupByKeyFn}.
+   */
+  private List<KV<BeamSQLRow, BeamSQLRow>> prepareResultOfAggregationGroupByKeyFn() {
+    return Arrays.asList(
+        KV.of(new BeamSQLRow(keyType, Arrays.<Object>asList(inputRows.get(0).getInteger(0))),
+            inputRows.get(0)),
+        KV.of(new BeamSQLRow(keyType, Arrays.<Object>asList(inputRows.get(1).getInteger(0))),
+            inputRows.get(1)),
+        KV.of(new BeamSQLRow(keyType, Arrays.<Object>asList(inputRows.get(2).getInteger(0))),
+            inputRows.get(2)),
+        KV.of(new BeamSQLRow(keyType, Arrays.<Object>asList(inputRows.get(3).getInteger(0))),
+            inputRows.get(3)));
+  }
+
+  /**
+   * expected results after {@link BeamAggregationTransforms.AggregationCombineFn}.
+   */
+  private List<KV<BeamSQLRow, BeamSQLRow>> prepareResultOfAggregationCombineFn()
+      throws ParseException {
+    BeamSQLRecordType aggPartType = initTypeOfSqlRow(
+        Arrays.asList(KV.of("count", SqlTypeName.BIGINT),
+
+            KV.of("sum1", SqlTypeName.BIGINT), KV.of("avg1", SqlTypeName.BIGINT),
+            KV.of("max1", SqlTypeName.BIGINT), KV.of("min1", SqlTypeName.BIGINT),
+
+            KV.of("sum2", SqlTypeName.SMALLINT), KV.of("avg2", SqlTypeName.SMALLINT),
+            KV.of("max2", SqlTypeName.SMALLINT), KV.of("min2", SqlTypeName.SMALLINT),
+
+            KV.of("sum3", SqlTypeName.TINYINT), KV.of("avg3", SqlTypeName.TINYINT),
+            KV.of("max3", SqlTypeName.TINYINT), KV.of("min3", SqlTypeName.TINYINT),
+
+            KV.of("sum4", SqlTypeName.FLOAT), KV.of("avg4", SqlTypeName.FLOAT),
+            KV.of("max4", SqlTypeName.FLOAT), KV.of("min4", SqlTypeName.FLOAT),
+
+            KV.of("sum5", SqlTypeName.DOUBLE), KV.of("avg5", SqlTypeName.DOUBLE),
+            KV.of("max5", SqlTypeName.DOUBLE), KV.of("min5", SqlTypeName.DOUBLE),
+
+            KV.of("max7", SqlTypeName.TIMESTAMP), KV.of("min7", SqlTypeName.TIMESTAMP),
+
+            KV.of("sum8", SqlTypeName.INTEGER), KV.of("avg8", SqlTypeName.INTEGER),
+            KV.of("max8", SqlTypeName.INTEGER), KV.of("min8", SqlTypeName.INTEGER)
+            ));
+    return Arrays.asList(
+            KV.of(new BeamSQLRow(keyType, Arrays.<Object>asList(inputRows.get(0).getInteger(0))),
+                new BeamSQLRow(aggPartType, Arrays.<Object>asList(
+                    4L,
+                    10000L, 2500L, 4000L, 1000L,
+                    (short) 10, (short) 2, (short) 4, (short) 1,
+                    (byte) 10, (byte) 2, (byte) 4, (byte) 1,
+                    10.0F, 2.5F, 4.0F, 1.0F,
+                    10.0, 2.5, 4.0, 1.0,
+                    format.parse("2017-01-01 02:04:03"), format.parse("2017-01-01 01:01:03"),
+                    10, 2, 4, 1
+                    )))
+            );
+  }
+
+  /**
+   * Row type of final output row.
+   */
+  private RelDataType prepareFinalRowType() {
+    FieldInfoBuilder builder = BeamQueryPlanner.TYPE_FACTORY.builder();
+    List<KV<String, SqlTypeName>> columnMetadata =
+        Arrays.asList(KV.of("f_int", SqlTypeName.INTEGER), KV.of("count", SqlTypeName.BIGINT),
+
+        KV.of("sum1", SqlTypeName.BIGINT), KV.of("avg1", SqlTypeName.BIGINT),
+        KV.of("max1", SqlTypeName.BIGINT), KV.of("min1", SqlTypeName.BIGINT),
+
+        KV.of("sum2", SqlTypeName.SMALLINT), KV.of("avg2", SqlTypeName.SMALLINT),
+        KV.of("max2", SqlTypeName.SMALLINT), KV.of("min2", SqlTypeName.SMALLINT),
+
+        KV.of("sum3", SqlTypeName.TINYINT), KV.of("avg3", SqlTypeName.TINYINT),
+        KV.of("max3", SqlTypeName.TINYINT), KV.of("min3", SqlTypeName.TINYINT),
+
+        KV.of("sum4", SqlTypeName.FLOAT), KV.of("avg4", SqlTypeName.FLOAT),
+        KV.of("max4", SqlTypeName.FLOAT), KV.of("min4", SqlTypeName.FLOAT),
+
+        KV.of("sum5", SqlTypeName.DOUBLE), KV.of("avg5", SqlTypeName.DOUBLE),
+        KV.of("max5", SqlTypeName.DOUBLE), KV.of("min5", SqlTypeName.DOUBLE),
+
+        KV.of("max7", SqlTypeName.TIMESTAMP), KV.of("min7", SqlTypeName.TIMESTAMP),
+
+        KV.of("sum8", SqlTypeName.INTEGER), KV.of("avg8", SqlTypeName.INTEGER),
+        KV.of("max8", SqlTypeName.INTEGER), KV.of("min8", SqlTypeName.INTEGER)
+        );
+    for (KV<String, SqlTypeName> cm : columnMetadata) {
+      builder.add(cm.getKey(), cm.getValue());
+    }
+    return builder.build();
+  }
+
+  /**
+   * expected results after {@link BeamAggregationTransforms.MergeAggregationRecord}.
+   */
+  private BeamSQLRow prepareResultOfMergeAggregationRecord() throws ParseException {
+    return new BeamSQLRow(BeamSQLRecordType.from(prepareFinalRowType()), Arrays.<Object>asList(
+        1, 4L,
+        10000L, 2500L, 4000L, 1000L,
+        (short) 10, (short) 2, (short) 4, (short) 1,
+        (byte) 10, (byte) 2, (byte) 4, (byte) 1,
+        10.0F, 2.5F, 4.0F, 1.0F,
+        10.0, 2.5, 4.0, 1.0,
+        format.parse("2017-01-01 02:04:03"), format.parse("2017-01-01 01:01:03"),
+        10, 2, 4, 1
+        ));
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamTransformBaseTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamTransformBaseTest.java
new file mode 100644
index 000000000000..820d7f57da62
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamTransformBaseTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.schema.transform;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.BeforeClass;
+
+/**
+ * shared methods to test PTransforms which execute Beam SQL steps.
+ *
+ */
+public class BeamTransformBaseTest {
+  public static DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+  public static  BeamSQLRecordType inputRowType;
+  public static List<BeamSQLRow> inputRows;
+
+  @BeforeClass
+  public static void prepareInput() throws NumberFormatException, ParseException{
+    List<KV<String, SqlTypeName>> columnMetadata = Arrays.asList(
+        KV.of("f_int", SqlTypeName.INTEGER), KV.of("f_long", SqlTypeName.BIGINT),
+        KV.of("f_short", SqlTypeName.SMALLINT), KV.of("f_byte", SqlTypeName.TINYINT),
+        KV.of("f_float", SqlTypeName.FLOAT), KV.of("f_double", SqlTypeName.DOUBLE),
+        KV.of("f_string", SqlTypeName.VARCHAR), KV.of("f_timestamp", SqlTypeName.TIMESTAMP),
+        KV.of("f_int2", SqlTypeName.INTEGER)
+        );
+    inputRowType = initTypeOfSqlRow(columnMetadata);
+    inputRows = Arrays.asList(
+        initBeamSqlRow(columnMetadata,
+            Arrays.<Object>asList(1, 1000L, Short.valueOf("1"), Byte.valueOf("1"), 1.0F, 1.0,
+                "string_row1", format.parse("2017-01-01 01:01:03"), 1)),
+        initBeamSqlRow(columnMetadata,
+            Arrays.<Object>asList(1, 2000L, Short.valueOf("2"), Byte.valueOf("2"), 2.0F, 2.0,
+                "string_row2", format.parse("2017-01-01 01:02:03"), 2)),
+        initBeamSqlRow(columnMetadata,
+            Arrays.<Object>asList(1, 3000L, Short.valueOf("3"), Byte.valueOf("3"), 3.0F, 3.0,
+                "string_row3", format.parse("2017-01-01 01:03:03"), 3)),
+        initBeamSqlRow(columnMetadata, Arrays.<Object>asList(1, 4000L, Short.valueOf("4"),
+            Byte.valueOf("4"), 4.0F, 4.0, "string_row4", format.parse("2017-01-01 02:04:03"), 4)));
+  }
+
+  /**
+   * create a {@code BeamSQLRecordType} for given column metadata.
+   */
+  public static BeamSQLRecordType initTypeOfSqlRow(List<KV<String, SqlTypeName>> columnMetadata){
+    FieldInfoBuilder builder = BeamQueryPlanner.TYPE_FACTORY.builder();
+    for (KV<String, SqlTypeName> cm : columnMetadata) {
+      builder.add(cm.getKey(), cm.getValue());
+    }
+    return BeamSQLRecordType.from(builder.build());
+  }
+
+  /**
+   * Create an empty row with given column metadata.
+   */
+  public static BeamSQLRow initBeamSqlRow(List<KV<String, SqlTypeName>> columnMetadata) {
+    return initBeamSqlRow(columnMetadata, Arrays.asList());
+  }
+
+  /**
+   * Create a row with given column metadata, and values for each column.
+   *
+   */
+  public static BeamSQLRow initBeamSqlRow(List<KV<String, SqlTypeName>> columnMetadata,
+      List<Object> rowValues){
+    BeamSQLRecordType rowType = initTypeOfSqlRow(columnMetadata);
+
+    return new BeamSQLRow(rowType, rowValues);
+  }
+
+}

From b50ddedb17b08aa4f2fd0672b47c461084e2dff3 Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Wed, 10 May 2017 17:19:48 +0800
Subject: [PATCH 376/387] [BEAM-2195] Implement conditional operator (CASE)

---
 .../sql/interpreter/BeamSQLFnExecutor.java    |  3 +
 .../operator/BeamSqlCaseExpression.java       | 64 +++++++++++++
 .../interpreter/BeamSQLFnExecutorTest.java    | 11 +++
 .../operator/BeamSqlCaseExpressionTest.java   | 94 +++++++++++++++++++
 4 files changed, 172 insertions(+)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpression.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpressionTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
index 4ae7b337c655..4b7af2a66bd8 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -22,6 +22,7 @@
 
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlAndExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlCaseExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlEqualExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlInputRefExpression;
@@ -159,6 +160,8 @@ static BeamSqlExpression buildExpression(RexNode rexNode) {
         case "INITCAP":
           return new BeamSqlInitCapExpression(subExps);
 
+        case "CASE":
+          return new BeamSqlCaseExpression(subExps);
 
         case "IS NULL":
           return new BeamSqlIsNullExpression(subExps.get(0));
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpression.java
new file mode 100644
index 000000000000..d108abd3f997
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpression.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ *  {@code BeamSqlCaseExpression} represents CASE, NULLIF, COALESCE in SQL.
+ */
+public class BeamSqlCaseExpression extends BeamSqlExpression {
+  public BeamSqlCaseExpression(List<BeamSqlExpression> operands) {
+    // the return type of CASE is the type of the `else` condition
+    super(operands, operands.get(operands.size() - 1).getOutputType());
+  }
+
+  @Override public boolean accept() {
+    // `when`-`then` pair + `else`
+    if (operands.size() % 2 != 1) {
+      return false;
+    }
+
+    for (int i = 0; i < operands.size() - 1; i += 2) {
+      if (opType(i) != SqlTypeName.BOOLEAN) {
+        return false;
+      } else if (opType(i + 1) != outputType) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    for (int i = 0; i < operands.size() - 1; i += 2) {
+      if (opValueEvaluated(i, inputRecord)) {
+        return BeamSqlPrimitive.of(
+            outputType,
+            opValueEvaluated(i + 1, inputRecord)
+        );
+      }
+    }
+    return BeamSqlPrimitive.of(outputType,
+        opValueEvaluated(operands.size() - 1, inputRecord));
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
index d7379fce8d52..ba9f525165dc 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutorTest.java
@@ -23,6 +23,7 @@
 import java.util.Arrays;
 
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlAndExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlCaseExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlEqualExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlInputRefExpression;
@@ -253,5 +254,15 @@ public void testBuildExpression_string()  {
     );
     exp = BeamSQLFnExecutor.buildExpression(rexNode);
     assertTrue(exp instanceof BeamSqlOverlayExpression);
+
+    rexNode = rexBuilder.makeCall(SqlStdOperatorTable.CASE,
+        Arrays.asList(
+            rexBuilder.makeLiteral(true),
+            rexBuilder.makeLiteral("HELLO"),
+            rexBuilder.makeLiteral("HELLO")
+        )
+    );
+    exp = BeamSQLFnExecutor.buildExpression(rexNode);
+    assertTrue(exp instanceof BeamSqlCaseExpression);
   }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpressionTest.java
new file mode 100644
index 000000000000..06b5073ae61d
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCaseExpressionTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlCaseExpression.
+ */
+public class BeamSqlCaseExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertTrue(new BeamSqlCaseExpression(operands).accept());
+
+    // even param count
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertFalse(new BeamSqlCaseExpression(operands).accept());
+
+    // `when` type error
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "error"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertFalse(new BeamSqlCaseExpression(operands).accept());
+
+    // `then` type mixing
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 10));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    assertFalse(new BeamSqlCaseExpression(operands).accept());
+
+  }
+
+  @Test public void evaluate() throws Exception {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertEquals("hello", new BeamSqlCaseExpression(operands)
+        .evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertEquals("world", new BeamSqlCaseExpression(operands)
+        .evaluate(record).getValue());
+
+    operands.clear();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, false));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello1"));
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
+    assertEquals("hello1", new BeamSqlCaseExpression(operands)
+        .evaluate(record).getValue());
+  }
+}

From 7995dfc957406b0eb98181f10b28c1af65d10f0d Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Fri, 12 May 2017 21:14:29 -0700
Subject: [PATCH 377/387] [BEAM-2234] Change return type of buildBeamPipeline
 to `PCollection<BeamSQLRow>`

---
 .../dsls/sql/planner/BeamPipelineCreator.java   | 15 ---------------
 .../beam/dsls/sql/planner/BeamQueryPlanner.java |  4 +++-
 .../beam/dsls/sql/rel/BeamAggregationRel.java   | 12 +++++-------
 .../apache/beam/dsls/sql/rel/BeamFilterRel.java | 14 ++++++--------
 .../apache/beam/dsls/sql/rel/BeamIOSinkRel.java | 17 ++++++++---------
 .../beam/dsls/sql/rel/BeamIOSourceRel.java      |  8 +++-----
 .../beam/dsls/sql/rel/BeamProjectRel.java       | 13 +++++--------
 .../apache/beam/dsls/sql/rel/BeamRelNode.java   | 11 +++++------
 8 files changed, 35 insertions(+), 59 deletions(-)

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
index 1d7cfd1b5704..98ccb57e939b 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
@@ -19,9 +19,6 @@
 
 import java.util.Map;
 
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
@@ -32,7 +29,6 @@
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.values.PCollection;
 
 /**
  * {@link BeamPipelineCreator} converts a {@link BeamRelNode} tree, into a Beam
@@ -41,7 +37,6 @@
  */
 public class BeamPipelineCreator {
   private Map<String, BaseBeamTable> sourceTables;
-  private Queue<PCollection<BeamSQLRow>> upStreamQueue;
 
   private PipelineOptions options;
 
@@ -56,22 +51,12 @@ public BeamPipelineCreator(Map<String, BaseBeamTable> sourceTables) {
         .as(PipelineOptions.class); // FlinkPipelineOptions.class
     options.setJobName("BeamPlanCreator");
 
-    upStreamQueue = new ConcurrentLinkedQueue<>();
-
     pipeline = Pipeline.create(options);
     CoderRegistry cr = pipeline.getCoderRegistry();
     cr.registerCoder(BeamSQLRow.class, BeamSqlRowCoder.of());
     cr.registerCoder(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
   }
 
-  public PCollection<BeamSQLRow> popUpstream() {
-    return upStreamQueue.poll();
-  }
-
-  public void pushUpstream(PCollection<BeamSQLRow> upstream) {
-    this.upStreamQueue.add(upstream);
-  }
-
   public Map<String, BaseBeamTable> getSourceTables() {
     return sourceTables;
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
index 935dae768401..29b3f1d09eed 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
@@ -112,7 +112,9 @@ public Pipeline compileBeamPipeline(String sqlStatement) throws Exception {
     BeamRelNode relNode = convertToBeamRel(sqlStatement);
 
     BeamPipelineCreator planCreator = new BeamPipelineCreator(sourceTables);
-    return relNode.buildBeamPipeline(planCreator);
+    relNode.buildBeamPipeline(planCreator);
+
+    return planCreator.getPipeline();
   }
 
   /**
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
index ab98cc470a1c..3e147aade901 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java
@@ -23,7 +23,6 @@
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.dsls.sql.transform.BeamAggregationTransforms;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -68,13 +67,13 @@ public BeamAggregationRel(RelOptCluster cluster, RelTraitSet traits
   }
 
   @Override
-  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+  public PCollection<BeamSQLRow> buildBeamPipeline(BeamPipelineCreator planCreator)
+      throws Exception {
     RelNode input = getInput();
-    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
-
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
+    PCollection<BeamSQLRow> upstream =
+        BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
     if (windowFieldIdx != -1) {
       upstream = upstream.apply("assignEventTimestamp", WithTimestamps
           .<BeamSQLRow>of(new BeamAggregationTransforms.WindowTimestampFn(windowFieldIdx)));
@@ -105,9 +104,8 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
     PCollection<BeamSQLRow> mergedStream = aggregatedStream.apply("mergeRecord",
         ParDo.of(new BeamAggregationTransforms.MergeAggregationRecord(
             BeamSQLRecordType.from(getRowType()), getAggCallList())));
-    planCreator.pushUpstream(mergedStream);
 
-    return planCreator.getPipeline();
+    return mergedStream;
   }
 
   @Override
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
index e1c5b3e7ebdc..f2c1bba337c2 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java
@@ -23,7 +23,6 @@
 import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.dsls.sql.transform.BeamSQLFilterFn;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.plan.RelOptCluster;
@@ -49,23 +48,22 @@ public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
   }
 
   @Override
-  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+  public PCollection<BeamSQLRow> buildBeamPipeline(BeamPipelineCreator planCreator)
+      throws Exception {
 
     RelNode input = getInput();
-    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
 
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
+    PCollection<BeamSQLRow> upstream = BeamSQLRelUtils.getBeamRelInput(input)
+        .buildBeamPipeline(planCreator);
 
     BeamSQLExpressionExecutor executor = new BeamSQLFnExecutor(this);
 
-    PCollection<BeamSQLRow> projectStream = upstream.apply(stageName,
+    PCollection<BeamSQLRow> filterStream = upstream.apply(stageName,
         ParDo.of(new BeamSQLFilterFn(getRelTypeName(), executor)));
 
-    planCreator.pushUpstream(projectStream);
-
-    return planCreator.getPipeline();
+    return filterStream;
   }
 
 }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
index f38b9e16c501..bc94ab8102e1 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java
@@ -24,7 +24,6 @@
 import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
@@ -52,15 +51,17 @@ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
         getOperation(), getUpdateColumnList(), getSourceExpressionList(), isFlattened());
   }
 
+  /**
+   * Note that {@code BeamIOSinkRel} returns the input PCollection.
+   */
   @Override
-  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
-
+  public PCollection<BeamSQLRow> buildBeamPipeline(BeamPipelineCreator planCreator)
+      throws Exception {
     RelNode input = getInput();
-    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
-
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
+    PCollection<BeamSQLRow> upstream =
+        BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
 
     String sourceName = Joiner.on('.').join(getTable().getQualifiedName());
 
@@ -68,9 +69,7 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
 
     upstream.apply(stageName, targetTable.buildIOWriter());
 
-    planCreator.setHasPersistent(true);
-
-    return planCreator.getPipeline();
+    return upstream;
   }
 
 }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
index 35382730881b..61f53eb14893 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
@@ -23,7 +23,6 @@
 import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
@@ -41,7 +40,8 @@ public BeamIOSourceRel(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable
   }
 
   @Override
-  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+  public PCollection<BeamSQLRow> buildBeamPipeline(BeamPipelineCreator planCreator)
+      throws Exception {
 
     String sourceName = Joiner.on('.').join(getTable().getQualifiedName()).replace(".(STREAM)", "");
 
@@ -52,9 +52,7 @@ public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Except
     PCollection<BeamSQLRow> sourceStream = planCreator.getPipeline().apply(stageName,
         sourceTable.buildIOReader());
 
-    planCreator.pushUpstream(sourceStream);
-
-    return planCreator.getPipeline();
+    return sourceStream;
   }
 
 }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
index 65f5b2003dba..954868d4b77b 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java
@@ -26,7 +26,6 @@
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.dsls.sql.transform.BeamSQLProjectFn;
-import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.plan.RelOptCluster;
@@ -61,22 +60,20 @@ public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> projects,
   }
 
   @Override
-  public Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception {
+  public PCollection<BeamSQLRow> buildBeamPipeline(BeamPipelineCreator planCreator)
+      throws Exception {
     RelNode input = getInput();
-    BeamSQLRelUtils.getBeamRelInput(input).buildBeamPipeline(planCreator);
-
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> upstream = planCreator.popUpstream();
+    PCollection<BeamSQLRow> upstream = BeamSQLRelUtils.getBeamRelInput(input)
+        .buildBeamPipeline(planCreator);
 
     BeamSQLExpressionExecutor executor = new BeamSQLFnExecutor(this);
 
     PCollection<BeamSQLRow> projectStream = upstream.apply(stageName, ParDo
         .of(new BeamSQLProjectFn(getRelTypeName(), executor, BeamSQLRecordType.from(rowType))));
 
-    planCreator.pushUpstream(projectStream);
-
-    return planCreator.getPipeline();
+    return projectStream;
 
   }
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
index e50d71a282d3..ff2b5b652b2f 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
@@ -18,7 +18,8 @@
 package org.apache.beam.dsls.sql.rel;
 
 import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
-import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.rel.RelNode;
 
 /**
@@ -29,10 +30,8 @@
 public interface BeamRelNode extends RelNode {
 
   /**
-   * A {@link BeamRelNode} is a recursive structure, the
-   * {@link BeamPipelineCreator} visits it with a DFS(Depth-First-Search)
-   * algorithm.
-   *
+   * {@code #buildBeamPipeline(BeamPipelineCreator)} applies a transform to upstream,
+   * and generate an output {@code PCollection}.
    */
-  Pipeline buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception;
+  PCollection<BeamSQLRow> buildBeamPipeline(BeamPipelineCreator planCreator) throws Exception;
 }

From f4d1aefe7fecd18345c23be9b409aece7a665e8e Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Sat, 13 May 2017 01:07:01 +0800
Subject: [PATCH 378/387] [BEAM-2149] Improved kafka table implemention.

1. use robust CSV library to parse & print.
2. support different data types rather than just `String`.
3. a little cleanup for TextTable (to extract common methods).
---
 .../beam/dsls/sql/schema/BeamTableUtils.java  | 104 ++++++++++++++++
 .../sql/schema/kafka/BeamKafkaCSVTable.java   |  53 ++++----
 .../schema/text/BeamTextCSVTableIOReader.java |  66 +---------
 .../schema/text/BeamTextCSVTableIOWriter.java |  22 +---
 .../schema/kafka/BeamKafkaCSVTableTest.java   | 113 ++++++++++++++++++
 .../sql/schema/text/BeamTextCSVTableTest.java |   2 -
 6 files changed, 245 insertions(+), 115 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTableTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
new file mode 100644
index 000000000000..bc622c2de9e9
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.CSVRecord;
+
+/**
+ * Utility methods for working with {@code BeamTable}.
+ */
+public final class BeamTableUtils {
+  public static BeamSQLRow csvLine2BeamSQLRow(
+      CSVFormat csvFormat,
+      String line,
+      BeamSQLRecordType beamSqlRecordType) {
+    BeamSQLRow row = new BeamSQLRow(beamSqlRecordType);
+    try (StringReader reader = new StringReader(line)) {
+      CSVParser parser = csvFormat.parse(reader);
+      CSVRecord rawRecord = parser.getRecords().get(0);
+
+      if (rawRecord.size() != beamSqlRecordType.size()) {
+        throw new IllegalArgumentException(String.format(
+            "Expect %d fields, but actually %d", line,
+            beamSqlRecordType.size(), rawRecord.size()
+        ));
+      } else {
+        for (int idx = 0; idx < beamSqlRecordType.size(); idx++) {
+          String raw = rawRecord.get(idx);
+          addFieldWithAutoTypeCasting(row, idx, raw);
+        }
+      }
+    } catch (IOException e) {
+      throw new IllegalArgumentException("decodeRecord failed!", e);
+    }
+    return row;
+  }
+
+  public static String beamSQLRow2CsvLine(BeamSQLRow row, CSVFormat csvFormat) {
+    StringWriter writer = new StringWriter();
+    try (CSVPrinter printer = csvFormat.print(writer)) {
+      for (int i = 0; i < row.size(); i++) {
+        printer.print(row.getFieldValue(i).toString());
+      }
+      printer.println();
+    } catch (IOException e) {
+      throw new IllegalArgumentException("encodeRecord failed!", e);
+    }
+    return writer.toString();
+  }
+
+  public static void addFieldWithAutoTypeCasting(BeamSQLRow row, int idx, String raw) {
+    SqlTypeName columnType = row.getDataType().getFieldsType().get(idx);
+    switch (columnType) {
+      case TINYINT:
+        row.addField(idx, Byte.valueOf(raw));
+        break;
+      case SMALLINT:
+        row.addField(idx, Short.valueOf(raw));
+        break;
+      case INTEGER:
+        row.addField(idx, Integer.valueOf(raw));
+        break;
+      case BIGINT:
+        row.addField(idx, Long.valueOf(raw));
+        break;
+      case FLOAT:
+        row.addField(idx, Float.valueOf(raw));
+        break;
+      case DOUBLE:
+        row.addField(idx, Double.valueOf(raw));
+        break;
+      case VARCHAR:
+        row.addField(idx, raw);
+        break;
+      default:
+        throw new BeamSqlUnsupportedException(
+            String.format("Column type %s is not supported yet!", columnType));
+    }
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
index 0f40f33403de..127870c3c33b 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTable.java
@@ -17,7 +17,11 @@
  */
 package org.apache.beam.dsls.sql.schema.kafka;
 
+import static org.apache.beam.dsls.sql.schema.BeamTableUtils.beamSQLRow2CsvLine;
+import static org.apache.beam.dsls.sql.schema.BeamTableUtils.csvLine2BeamSQLRow;
+
 import java.util.List;
+
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -26,33 +30,35 @@
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.rel.type.RelProtoDataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.csv.CSVFormat;
 
 /**
  * A Kafka topic that saves records as CSV format.
  *
  */
 public class BeamKafkaCSVTable extends BeamKafkaTable {
-
-  public static final String DELIMITER = ",";
-  private static final Logger LOG = LoggerFactory.getLogger(BeamKafkaCSVTable.class);
-
+  private CSVFormat csvFormat;
   public BeamKafkaCSVTable(RelProtoDataType protoRowType, String bootstrapServers,
       List<String> topics) {
+    this(protoRowType, bootstrapServers, topics, CSVFormat.DEFAULT);
+  }
+
+  public BeamKafkaCSVTable(RelProtoDataType protoRowType, String bootstrapServers,
+      List<String> topics, CSVFormat format) {
     super(protoRowType, bootstrapServers, topics);
+    this.csvFormat = format;
   }
 
   @Override
   public PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamSQLRow>>
       getPTransformForInput() {
-    return new CsvRecorderDecoder(beamSqlRecordType);
+    return new CsvRecorderDecoder(beamSqlRecordType, csvFormat);
   }
 
   @Override
   public PTransform<PCollection<BeamSQLRow>, PCollection<KV<byte[], byte[]>>>
       getPTransformForOutput() {
-    return new CsvRecorderEncoder(beamSqlRecordType);
+    return new CsvRecorderEncoder(beamSqlRecordType, csvFormat);
   }
 
   /**
@@ -62,9 +68,10 @@ public BeamKafkaCSVTable(RelProtoDataType protoRowType, String bootstrapServers,
   public static class CsvRecorderDecoder
       extends PTransform<PCollection<KV<byte[], byte[]>>, PCollection<BeamSQLRow>> {
     private BeamSQLRecordType recordType;
-
-    public CsvRecorderDecoder(BeamSQLRecordType recordType) {
+    private CSVFormat format;
+    public CsvRecorderDecoder(BeamSQLRecordType recordType, CSVFormat format) {
       this.recordType = recordType;
+      this.format = format;
     }
 
     @Override
@@ -73,16 +80,7 @@ public PCollection<BeamSQLRow> expand(PCollection<KV<byte[], byte[]>> input) {
         @ProcessElement
         public void processElement(ProcessContext c) {
           String rowInString = new String(c.element().getValue());
-          String[] parts = rowInString.split(BeamKafkaCSVTable.DELIMITER);
-          if (parts.length != recordType.size()) {
-            LOG.error(String.format("invalid record: ", rowInString));
-          } else {
-            BeamSQLRow sourceRecord = new BeamSQLRow(recordType);
-            for (int idx = 0; idx < parts.length; ++idx) {
-              sourceRecord.addField(idx, parts[idx]);
-            }
-            c.output(sourceRecord);
-          }
+          c.output(csvLine2BeamSQLRow(format, rowInString, recordType));
         }
       }));
     }
@@ -95,9 +93,10 @@ public void processElement(ProcessContext c) {
   public static class CsvRecorderEncoder
       extends PTransform<PCollection<BeamSQLRow>, PCollection<KV<byte[], byte[]>>> {
     private BeamSQLRecordType recordType;
-
-    public CsvRecorderEncoder(BeamSQLRecordType recordType) {
+    private CSVFormat format;
+    public CsvRecorderEncoder(BeamSQLRecordType recordType, CSVFormat format) {
       this.recordType = recordType;
+      this.format = format;
     }
 
     @Override
@@ -106,17 +105,9 @@ public PCollection<KV<byte[], byte[]>> expand(PCollection<BeamSQLRow> input) {
         @ProcessElement
         public void processElement(ProcessContext c) {
           BeamSQLRow in = c.element();
-          StringBuffer sb = new StringBuffer();
-          for (int idx = 0; idx < in.size(); ++idx) {
-            sb.append(DELIMITER);
-            sb.append(in.getFieldValue(idx).toString());
-          }
-          c.output(KV.of(new byte[] {}, sb.substring(1).getBytes()));
+          c.output(KV.of(new byte[] {}, beamSQLRow2CsvLine(in, format).getBytes()));
         }
       }));
-
     }
-
   }
-
 }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
index cf7c095f7dfd..3c031cead6b8 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
@@ -18,11 +18,10 @@
 
 package org.apache.beam.dsls.sql.schema.text;
 
-import java.io.IOException;
+import static org.apache.beam.dsls.sql.schema.BeamTableUtils.csvLine2BeamSQLRow;
+
 import java.io.Serializable;
-import java.io.StringReader;
 
-import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.io.TextIO;
@@ -31,12 +30,7 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
-import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVRecord;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * IOReader for {@code BeamTextCSVTable}.
@@ -44,7 +38,6 @@
 public class BeamTextCSVTableIOReader
     extends PTransform<PBegin, PCollection<BeamSQLRow>>
     implements Serializable {
-  private static final Logger LOG = LoggerFactory.getLogger(BeamTextCSVTableIOReader.class);
   private String filePattern;
   protected BeamSQLRecordType beamSqlRecordType;
   protected CSVFormat csvFormat;
@@ -63,61 +56,8 @@ public PCollection<BeamSQLRow> expand(PBegin input) {
           @ProcessElement
           public void processElement(ProcessContext ctx) {
             String str = ctx.element();
-
-            try (StringReader reader = new StringReader(str)) {
-              CSVRecord rawRecord = null;
-              try {
-                CSVParser parser = csvFormat.parse(reader);
-                rawRecord = parser.getRecords().get(0);
-              } catch (IOException e) {
-                throw new IllegalArgumentException("Invalid text filePattern: " + filePattern, e);
-              }
-
-              BeamSQLRow row = new BeamSQLRow(beamSqlRecordType);
-              if (rawRecord.size() != beamSqlRecordType.size()) {
-                throw new IllegalArgumentException(String.format(
-                    "Invalid filePattern: {}, expect %d fields, but actually %d", str,
-                    filePattern, beamSqlRecordType.size(), rawRecord.size()
-                ));
-              } else {
-                for (int idx = 0; idx < beamSqlRecordType.size(); idx++) {
-                  String raw = rawRecord.get(idx);
-                  addFieldWithAutoTypeCasting(row, idx, raw);
-                }
-                ctx.output(row);
-              }
-            }
+            ctx.output(csvLine2BeamSQLRow(csvFormat, str, beamSqlRecordType));
           }
         }));
   }
-
-  public void addFieldWithAutoTypeCasting(BeamSQLRow row, int idx, String raw) {
-    SqlTypeName columnType = row.getDataType().getFieldsType().get(idx);
-    switch (columnType) {
-      case TINYINT:
-        row.addField(idx, Byte.valueOf(raw));
-        break;
-      case SMALLINT:
-        row.addField(idx, Short.valueOf(raw));
-        break;
-      case INTEGER:
-        row.addField(idx, Integer.valueOf(raw));
-        break;
-      case BIGINT:
-        row.addField(idx, Long.valueOf(raw));
-        break;
-      case FLOAT:
-        row.addField(idx, Float.valueOf(raw));
-        break;
-      case DOUBLE:
-        row.addField(idx, Double.valueOf(raw));
-        break;
-      case VARCHAR:
-        row.addField(idx, raw);
-        break;
-      default:
-        throw new BeamSqlUnsupportedException(String.format(
-            "Column type %s is not supported yet!", columnType));
-    }
-  }
 }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
index 6104cd8e9491..eade842a7cf5 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
@@ -18,9 +18,9 @@
 
 package org.apache.beam.dsls.sql.schema.text;
 
-import java.io.IOException;
+import static org.apache.beam.dsls.sql.schema.BeamTableUtils.beamSQLRow2CsvLine;
+
 import java.io.Serializable;
-import java.io.StringWriter;
 
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
@@ -31,17 +31,12 @@
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVPrinter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * IOWriter for {@code BeamTextCSVTable}.
  */
 public class BeamTextCSVTableIOWriter extends PTransform<PCollection<BeamSQLRow>, PDone>
     implements Serializable {
-  private static final Logger LOG = LoggerFactory.getLogger(BeamTextCSVTableIOWriter.class);
-
   private String filePattern;
   protected BeamSQLRecordType beamSqlRecordType;
   protected CSVFormat csvFormat;
@@ -58,18 +53,7 @@ public BeamTextCSVTableIOWriter(BeamSQLRecordType beamSqlRecordType, String file
 
       @ProcessElement public void processElement(ProcessContext ctx) {
         BeamSQLRow row = ctx.element();
-        StringWriter writer = new StringWriter();
-
-        try (CSVPrinter printer = csvFormat.print(writer)) {
-          for (int i = 0; i < row.size(); i++) {
-            printer.print(row.getFieldValue(i).toString());
-          }
-          printer.println();
-        } catch (IOException e) {
-          throw new IllegalArgumentException("Invalid filePattern: " + filePattern, e);
-        }
-
-        ctx.output(writer.toString());
+        ctx.output(beamSQLRow2CsvLine(row, csvFormat));
       }
     })).apply(TextIO.Write.to(filePattern));
   }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTableTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTableTest.java
new file mode 100644
index 000000000000..d20af0ced0c8
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaCSVTableTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema.kafka;
+
+import java.io.Serializable;
+
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.commons.csv.CSVFormat;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for BeamKafkaCSVTable.
+ */
+public class BeamKafkaCSVTableTest {
+  @Rule
+  public TestPipeline pipeline = TestPipeline.create();
+  public static BeamSQLRow row1 = new BeamSQLRow(genRowType());
+  public static BeamSQLRow row2 = new BeamSQLRow(genRowType());
+
+  @BeforeClass
+  public static void setUp() {
+    row1.addField(0, 1L);
+    row1.addField(1, 1);
+    row1.addField(2, 1.0);
+
+    row2.addField(0, 2L);
+    row2.addField(1, 2);
+    row2.addField(2, 2.0);
+  }
+
+  @Test public void testCsvRecorderDecoder() throws Exception {
+    PCollection<BeamSQLRow> result = pipeline
+        .apply(
+            Create.of("1,\"1\",1.0", "2,2,2.0")
+        )
+        .apply(ParDo.of(new String2KvBytes()))
+        .apply(
+            new BeamKafkaCSVTable.CsvRecorderDecoder(genRowType(), CSVFormat.DEFAULT)
+        );
+
+    PAssert.that(result).containsInAnyOrder(row1, row2);
+
+    pipeline.run();
+  }
+
+  @Test public void testCsvRecorderEncoder() throws Exception {
+    PCollection<BeamSQLRow> result = pipeline
+        .apply(
+            Create.of(row1, row2)
+        )
+        .apply(
+            new BeamKafkaCSVTable.CsvRecorderEncoder(genRowType(), CSVFormat.DEFAULT)
+        ).apply(
+            new BeamKafkaCSVTable.CsvRecorderDecoder(genRowType(), CSVFormat.DEFAULT)
+        );
+
+    PAssert.that(result).containsInAnyOrder(row1, row2);
+
+    pipeline.run();
+  }
+
+  private static BeamSQLRecordType genRowType() {
+    return BeamSQLRecordType.from(
+        new RelProtoDataType() {
+          @Override public RelDataType apply(RelDataTypeFactory a0) {
+            return a0.builder()
+                .add("order_id", SqlTypeName.BIGINT)
+                .add("site_id", SqlTypeName.INTEGER)
+                .add("price", SqlTypeName.DOUBLE)
+                .build();
+          }
+        }.apply(BeamQueryPlanner.TYPE_FACTORY));
+  }
+
+  private static class String2KvBytes extends DoFn<String, KV<byte[], byte[]>>
+      implements Serializable {
+    @ProcessElement
+    public void processElement(ProcessContext ctx) {
+      ctx.output(KV.of(new byte[] {}, ctx.element().getBytes()));
+    }
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
index e06f8da4f8a9..3bc29e41ff20 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
@@ -31,7 +31,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
@@ -75,7 +74,6 @@ public class BeamTextCSVTableTest {
       add(buildRow(data));
     }
   }};
-  private static ConcurrentLinkedQueue<Object[]> actualData = new ConcurrentLinkedQueue<>();
 
   private static Path tempFolder;
   private static File readerSourceFile;

From 1483ca5521acdee8f3e34b6c95535d8487bf3f25 Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Sat, 13 May 2017 21:42:05 -0700
Subject: [PATCH 379/387] support UDF

---
 .../sql/interpreter/BeamSQLFnExecutor.java    | 13 ++-
 .../operator/BeamSqlUdfExpression.java        | 86 +++++++++++++++++++
 .../dsls/sql/planner/BeamQueryPlanner.java    |  5 +-
 .../beam/dsls/sql/planner/BeamSqlRunner.java  | 12 +++
 .../operator/BeamSqlUdfExpressionTest.java    | 51 +++++++++++
 .../sql/planner/BeamGroupByExplainTest.java   | 11 +++
 .../sql/planner/BeamGroupByPipelineTest.java  | 12 +++
 7 files changed, 188 insertions(+), 2 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpressionTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
index 4b7af2a66bd8..eb9fedf58842 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -35,6 +35,7 @@
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlNotEqualExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlOrExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlUdfExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlWindowEndExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlWindowExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlWindowStartExpression;
@@ -60,7 +61,9 @@
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.impl.ScalarFunctionImpl;
 import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
 
 /**
  * Executor based on {@link BeamSqlExpression} and {@link BeamSqlPrimitive}.
@@ -181,7 +184,15 @@ static BeamSqlExpression buildExpression(RexNode rexNode) {
       case "SESSION_END":
         return new BeamSqlWindowEndExpression();
       default:
-        throw new BeamSqlUnsupportedException("Operator: " + opName + " not supported yet!");
+        //handle UDF
+        if (((RexCall) rexNode).getOperator() instanceof SqlUserDefinedFunction) {
+          SqlUserDefinedFunction udf = (SqlUserDefinedFunction) ((RexCall) rexNode).getOperator();
+          ScalarFunctionImpl fn = (ScalarFunctionImpl) udf.getFunction();
+          return new BeamSqlUdfExpression(fn.method, subExps,
+              ((RexCall) rexNode).type.getSqlTypeName());
+        } else {
+          throw new BeamSqlUnsupportedException("Operator: " + opName + " not supported yet!");
+        }
       }
     } else {
       throw new BeamSqlUnsupportedException(
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java
new file mode 100644
index 000000000000..d6cf9874b30f
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * revoke a UDF function.
+ */
+public class BeamSqlUdfExpression extends BeamSqlExpression {
+  //as Method is not Serializable, need to keep class/method information, and rebuild it.
+  private transient Method method;
+  private String className;
+  private String methodName;
+  private List<String> paraClassName = new ArrayList<>();
+
+  public BeamSqlUdfExpression(Method method, List<BeamSqlExpression> subExps,
+      SqlTypeName sqlTypeName) {
+    super(subExps, sqlTypeName);
+    this.method = method;
+
+    this.className = method.getDeclaringClass().getName();
+    this.methodName = method.getName();
+    for (Class<?> c : method.getParameterTypes()) {
+      paraClassName.add(c.getName());
+    }
+  }
+
+  @Override
+  public boolean accept() {
+    return true;
+  }
+
+  @Override
+  public BeamSqlPrimitive evaluate(BeamSQLRow inputRecord) {
+    if (method == null) {
+      reConstructMethod();
+    }
+    try {
+      List<Object> paras = new ArrayList<>();
+      for (BeamSqlExpression e : getOperands()) {
+        paras.add(e.evaluate(inputRecord).getValue());
+      }
+
+      return BeamSqlPrimitive.of(getOutputType(),
+          method.invoke(null, paras.toArray(new Object[]{})));
+    } catch (Exception ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+
+  /**
+   * re-construct method from class/method.
+   */
+  private void reConstructMethod() {
+    try {
+      List<Class<?>> paraClass = new ArrayList<>();
+      for (String pc : paraClassName) {
+        paraClass.add(Class.forName(pc));
+      }
+      method = Class.forName(className).getMethod(methodName, paraClass.toArray(new Class<?>[] {}));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
index 29b3f1d09eed..9e4155592bff 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
@@ -48,6 +48,7 @@
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
 import org.apache.calcite.tools.FrameworkConfig;
 import org.apache.calcite.tools.Frameworks;
 import org.apache.calcite.tools.Planner;
@@ -83,7 +84,9 @@ public BeamQueryPlanner(SchemaPlus schema) {
     FrameworkConfig config = Frameworks.newConfigBuilder()
         .parserConfig(SqlParser.configBuilder().setLex(Lex.MYSQL).build()).defaultSchema(schema)
         .traitDefs(traitDefs).context(Contexts.EMPTY_CONTEXT).ruleSets(BeamRuleSets.getRuleSets())
-        .costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM).build();
+        .costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM)
+        .operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables))
+        .build();
     this.planner = Frameworks.getPlanner(config);
 
     for (String t : schema.getTableNames()) {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
index 708c5073608e..95ba5a959f7f 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
@@ -24,6 +24,7 @@
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.schema.Schema;
 import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.impl.ScalarFunctionImpl;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.calcite.tools.Frameworks;
 import org.apache.calcite.tools.RelConversionException;
@@ -64,6 +65,17 @@ public void addTable(String tableName, BaseBeamTable table) {
     planner.getSourceTables().put(tableName, table);
   }
 
+  /**
+   * Add a UDF function.
+   *
+   * <p>There're two requirements for function {@code methodName}:<br>
+   * 1. It must be a STATIC method;<br>
+   * 2. For a primitive parameter, use its wrapper class and handle NULL properly;
+   */
+  public void addUDFFunction(String functionName, Class<?> className, String methodName){
+    schema.add(functionName, ScalarFunctionImpl.create(className, methodName));
+  }
+
   /**
    * submit as a Beam pipeline.
    *
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpressionTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpressionTest.java
new file mode 100644
index 000000000000..71ac5234899f
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpressionTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.dsls.sql.interpreter.BeamSQLFnExecutorTestBase;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlUdfExpression.
+ */
+public class BeamSqlUdfExpressionTest extends BeamSQLFnExecutorTestBase {
+
+  @Test
+  public void testUdf() throws NoSuchMethodException, SecurityException {
+    List<BeamSqlExpression> operands = new ArrayList<>();
+    operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 10));
+
+    BeamSqlUdfExpression exp = new BeamSqlUdfExpression(
+        UdfFn.class.getMethod("negative", Integer.class), operands, SqlTypeName.INTEGER);
+
+    Assert.assertEquals(-10, exp.evaluate(record).getValue());
+  }
+
+  /**
+   * UDF example.
+   */
+  public static final class UdfFn {
+    public static int negative(Integer number) {
+      return number == null ? 0 : 0 - number;
+    }
+  }
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
index 566c5748b7da..4f2b1ba91407 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.dsls.sql.planner;
 
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlUdfExpressionTest;
 import org.junit.Test;
 
 /**
@@ -90,4 +91,14 @@ public void testSessionExplain() throws Exception {
     String plan = runner.explainQuery(sql);
   }
 
+  /**
+   * Query with UDF.
+   */
+  @Test
+  public void testUdf() throws Exception {
+    runner.addUDFFunction("negative", BeamSqlUdfExpressionTest.UdfFn.class, "negative");
+    String sql = "select site_id, negative(site_id) as nsite_id from ORDER_DETAILS";
+
+    String plan = runner.explainQuery(sql);
+  }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
index d5f81250833d..71dcf73d6b71 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.dsls.sql.planner;
 
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlUdfExpressionTest;
 import org.apache.beam.sdk.Pipeline;
 import org.junit.Test;
 
@@ -91,4 +92,15 @@ public void testSessionExplain() throws Exception {
     Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
   }
 
+  /**
+   * Query with UDF.
+   */
+  @Test
+  public void testUdf() throws Exception {
+    runner.addUDFFunction("negative", BeamSqlUdfExpressionTest.UdfFn.class, "negative");
+    String sql = "select site_id, negative(site_id) as nsite_id from ORDER_DETAILS";
+
+    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+  }
+
 }

From a92d7ba5bd593ccbab256fcde39ade0f3f0b7ddc Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Sun, 14 May 2017 08:56:57 -0700
Subject: [PATCH 380/387] update JavaDoc.

---
 .../dsls/sql/interpreter/operator/BeamSqlUdfExpression.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java
index d6cf9874b30f..389a87e240c6 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlUdfExpression.java
@@ -24,7 +24,7 @@
 import org.apache.calcite.sql.type.SqlTypeName;
 
 /**
- * revoke a UDF function.
+ * invoke a UDF function.
  */
 public class BeamSqlUdfExpression extends BeamSqlExpression {
   //as Method is not Serializable, need to keep class/method information, and rebuild it.

From 6beb2913b8f9b838e9de40eda83bead9a85a3ddd Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Fri, 12 May 2017 01:07:18 +0800
Subject: [PATCH 381/387] [BEAM-2255] Implement ORDER BY

---
 .../beam/dsls/sql/planner/BeamRuleSets.java   |   3 +-
 .../apache/beam/dsls/sql/rel/BeamSortRel.java | 242 ++++++++++++++++++
 .../beam/dsls/sql/rule/BeamSortRule.java      |  52 ++++
 .../beam/dsls/sql/schema/BeamSQLRow.java      |   7 +
 .../dsls/sql/planner/MockedBeamSQLTable.java  |  68 ++++-
 .../beam/dsls/sql/rel/BeamSortRelTest.java    | 231 +++++++++++++++++
 6 files changed, 601 insertions(+), 2 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamSortRule.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
index acbd43f47ef0..2cac5aeda2b9 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
@@ -27,6 +27,7 @@
 import org.apache.beam.dsls.sql.rule.BeamIOSinkRule;
 import org.apache.beam.dsls.sql.rule.BeamIOSourceRule;
 import org.apache.beam.dsls.sql.rule.BeamProjectRule;
+import org.apache.beam.dsls.sql.rule.BeamSortRule;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.tools.RuleSet;
@@ -40,7 +41,7 @@ public class BeamRuleSets {
   private static final ImmutableSet<RelOptRule> calciteToBeamConversionRules = ImmutableSet
       .<RelOptRule>builder().add(BeamIOSourceRule.INSTANCE, BeamProjectRule.INSTANCE,
           BeamFilterRule.INSTANCE, BeamIOSinkRule.INSTANCE,
-          BeamAggregationRule.INSTANCE)
+          BeamAggregationRule.INSTANCE, BeamSortRule.INSTANCE)
       .build();
 
   public static RuleSet[] getRuleSets() {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
new file mode 100644
index 000000000000..3df2f343f1bb
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.rel;
+
+import java.io.Serializable;
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.schema.UnsupportedDataTypeException;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Top;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Sort} node.
+ *
+ * <p>Since Beam does not fully supported global sort we are using {@link Top} to implement
+ * the {@code Sort} algebra. The following types of ORDER BY are supported:
+
+ * <pre>{@code
+ *     select * from t order by id desc limit 10;
+ *     select * from t order by id desc limit 10, 5;
+ * }</pre>
+ *
+ * <p>but Order BY without a limit is NOT supported:
+ *
+ * <pre>{@code
+ *   select * from t order by id desc
+ * }</pre>
+ *
+ * <h3>Constraints</h3>
+ * <ul>
+ *   <li>Due to the constraints of {@link Top}, the result of a `ORDER BY LIMIT`
+ *   must fit into the memory of a single machine.</li>
+ *   <li>Since `WINDOW`(HOP, TUMBLE, SESSION etc) is always associated with `GroupBy`,
+ *   it does not make much sense to use `ORDER BY` with `WINDOW`.
+ *   </li>
+ * </ul>
+ */
+public class BeamSortRel extends Sort implements BeamRelNode {
+  private List<Integer> fieldIndices = new ArrayList<>();
+  private List<Boolean> orientation = new ArrayList<>();
+  private List<Boolean> nullsFirst = new ArrayList<>();
+
+  private int startIndex = 0;
+  private int count;
+
+  public BeamSortRel(
+      RelOptCluster cluster,
+      RelTraitSet traits,
+      RelNode child,
+      RelCollation collation,
+      RexNode offset,
+      RexNode fetch) {
+    super(cluster, traits, child, collation, offset, fetch);
+
+    List<RexNode> fieldExps = getChildExps();
+    RelCollationImpl collationImpl = (RelCollationImpl) collation;
+    List<RelFieldCollation> collations = collationImpl.getFieldCollations();
+    for (int i = 0; i < fieldExps.size(); i++) {
+      RexNode fieldExp = fieldExps.get(i);
+      RexInputRef inputRef = (RexInputRef) fieldExp;
+      fieldIndices.add(inputRef.getIndex());
+      orientation.add(collations.get(i).getDirection() == RelFieldCollation.Direction.ASCENDING);
+
+      RelFieldCollation.NullDirection rawNullDirection = collations.get(i).nullDirection;
+      if (rawNullDirection == RelFieldCollation.NullDirection.UNSPECIFIED) {
+        rawNullDirection = collations.get(i).getDirection().defaultNullDirection();
+      }
+      nullsFirst.add(rawNullDirection == RelFieldCollation.NullDirection.FIRST);
+    }
+
+    if (fetch == null) {
+      throw new BeamSqlUnsupportedException("ORDER BY without a LIMIT is not supported!");
+    }
+
+    RexLiteral fetchLiteral = (RexLiteral) fetch;
+    count = ((BigDecimal) fetchLiteral.getValue()).intValue();
+
+    if (offset != null) {
+      RexLiteral offsetLiteral = (RexLiteral) offset;
+      startIndex = ((BigDecimal) offsetLiteral.getValue()).intValue();
+    }
+  }
+
+  @Override public PCollection<BeamSQLRow> buildBeamPipeline(
+      BeamPipelineCreator planCreator) throws Exception {
+    RelNode input = getInput();
+    PCollection<BeamSQLRow> upstream = BeamSQLRelUtils.getBeamRelInput(input)
+        .buildBeamPipeline(planCreator);
+    Type windowType = upstream.getWindowingStrategy().getWindowFn()
+        .getWindowTypeDescriptor().getType();
+    if (!windowType.equals(GlobalWindow.class)) {
+      throw new BeamSqlUnsupportedException(
+          "`ORDER BY` is only supported for GlobalWindow, actual window: " + windowType);
+    }
+
+    BeamSQLRowComparator comparator = new BeamSQLRowComparator(fieldIndices, orientation,
+        nullsFirst);
+    // first find the top (offset + count)
+    PCollection<List<BeamSQLRow>> rawStream =
+        upstream.apply("extractTopOffsetAndFetch",
+            Top.of(startIndex + count, comparator).withoutDefaults());
+
+    // strip the `leading offset`
+    if (startIndex > 0) {
+      rawStream = rawStream.apply("stripLeadingOffset", ParDo.of(
+          new SubListFn<BeamSQLRow>(startIndex, startIndex + count)));
+    }
+
+    PCollection<BeamSQLRow> orderedStream = rawStream.apply(
+        "flatten", Flatten.<BeamSQLRow>iterables());
+    return orderedStream;
+  }
+
+  private static class SubListFn<T> extends DoFn<List<T>, List<T>> {
+    private int startIndex;
+    private int endIndex;
+
+    public SubListFn(int startIndex, int endIndex) {
+      this.startIndex = startIndex;
+      this.endIndex = endIndex;
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext ctx) {
+      ctx.output(ctx.element().subList(startIndex, endIndex));
+    }
+  }
+
+  @Override public Sort copy(RelTraitSet traitSet, RelNode newInput, RelCollation newCollation,
+      RexNode offset, RexNode fetch) {
+    return new BeamSortRel(getCluster(), traitSet, newInput, newCollation, offset, fetch);
+  }
+
+  private static class BeamSQLRowComparator implements Comparator<BeamSQLRow>, Serializable {
+    private List<Integer> fieldsIndices;
+    private List<Boolean> orientation;
+    private List<Boolean> nullsFirst;
+
+    public BeamSQLRowComparator(List<Integer> fieldsIndices,
+        List<Boolean> orientation,
+        List<Boolean> nullsFirst) {
+      this.fieldsIndices = fieldsIndices;
+      this.orientation = orientation;
+      this.nullsFirst = nullsFirst;
+    }
+
+    @Override public int compare(BeamSQLRow row1, BeamSQLRow row2) {
+      for (int i = 0; i < fieldsIndices.size(); i++) {
+        int fieldIndex = fieldsIndices.get(i);
+        int fieldRet = 0;
+        SqlTypeName fieldType = row1.getDataType().getFieldsType().get(fieldIndex);
+        // whether NULL should be ordered first or last(compared to non-null values) depends on
+        // what user specified in SQL(NULLS FIRST/NULLS LAST)
+        if (row1.isNull(fieldIndex) && row2.isNull(fieldIndex)) {
+          continue;
+        } else if (row1.isNull(fieldIndex) && !row2.isNull(fieldIndex)) {
+          fieldRet = -1 * (nullsFirst.get(i) ? -1 : 1);
+        } else if (!row1.isNull(fieldIndex) && row2.isNull(fieldIndex)) {
+          fieldRet = 1 * (nullsFirst.get(i) ? -1 : 1);
+        } else {
+          switch (fieldType) {
+            case TINYINT:
+              fieldRet = numberCompare(row1.getByte(fieldIndex), row2.getByte(fieldIndex));
+              break;
+            case SMALLINT:
+              fieldRet = numberCompare(row1.getShort(fieldIndex), row2.getShort(fieldIndex));
+              break;
+            case INTEGER:
+              fieldRet = numberCompare(row1.getInteger(fieldIndex), row2.getInteger(fieldIndex));
+              break;
+            case BIGINT:
+              fieldRet = numberCompare(row1.getLong(fieldIndex), row2.getLong(fieldIndex));
+              break;
+            case FLOAT:
+              fieldRet = numberCompare(row1.getFloat(fieldIndex), row2.getFloat(fieldIndex));
+              break;
+            case DOUBLE:
+              fieldRet = numberCompare(row1.getDouble(fieldIndex), row2.getDouble(fieldIndex));
+              break;
+            case VARCHAR:
+              fieldRet = row1.getString(fieldIndex).compareTo(row2.getString(fieldIndex));
+              break;
+            case DATE:
+              fieldRet = row1.getDate(fieldIndex).compareTo(row2.getDate(fieldIndex));
+              break;
+            default:
+              throw new UnsupportedDataTypeException(fieldType);
+          }
+        }
+
+        fieldRet *= (orientation.get(i) ? -1 : 1);
+        if (fieldRet != 0) {
+          return fieldRet;
+        }
+      }
+      return 0;
+    }
+  }
+
+  public static <T extends Number & Comparable> int numberCompare(T a, T b) {
+    return a.compareTo(b);
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamSortRule.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamSortRule.java
new file mode 100644
index 000000000000..d802e9d913a6
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamSortRule.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.rule;
+
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
+
+import org.apache.beam.dsls.sql.rel.BeamSortRel;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.logical.LogicalSort;
+
+/**
+ * {@code ConverterRule} to replace {@code Sort} with {@code BeamSortRel}.
+ */
+public class BeamSortRule extends ConverterRule {
+  public static final BeamSortRule INSTANCE = new BeamSortRule();
+  private BeamSortRule() {
+    super(LogicalSort.class, Convention.NONE,
+        BeamLogicalConvention.INSTANCE, "BeamSortRule");
+  }
+
+  @Override public RelNode convert(RelNode rel) {
+    Sort sort = (Sort) rel;
+    final RelNode input = sort.getInput();
+    return new BeamSortRel(
+        sort.getCluster(),
+        sort.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
+        convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)),
+        sort.getCollation(),
+        sort.offset,
+        sort.fetch
+    );
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
index 5bdd5d2afed3..7b6428ed2194 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
@@ -285,6 +285,13 @@ public List<Integer> getNullFields() {
     return nullFields;
   }
 
+  /**
+   * is the specified field NULL?
+   */
+  public boolean isNull(int idx) {
+    return nullFields.contains(idx);
+  }
+
   public Instant getWindowStart() {
     return windowStart;
   }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
index 611bd73a89dd..8ccb33212f9f 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
@@ -19,8 +19,10 @@
 
 import java.util.ArrayList;
 import java.util.List;
+
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamIOType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -29,7 +31,10 @@
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
 
 /**
  * A mock table use to check input/output.
@@ -50,6 +55,64 @@ public MockedBeamSQLTable withInputRecords(List<BeamSQLRow> inputRecords){
     return this;
   }
 
+  /**
+   * Convenient way to build a mocked table with mock data:
+   *
+   * <p>e.g.
+   *
+   * <pre>{@code
+   * MockedBeamSQLTable
+   *   .of(SqlTypeName.BIGINT, "order_id",
+   *       SqlTypeName.INTEGER, "site_id",
+   *       SqlTypeName.DOUBLE, "price",
+   *       SqlTypeName.TIMESTAMP, "order_time",
+   *
+   *       1L, 2, 1.0, new Date(),
+   *       1L, 1, 2.0, new Date(),
+   *       2L, 4, 3.0, new Date(),
+   *       2L, 1, 4.0, new Date(),
+   *       5L, 5, 5.0, new Date(),
+   *       6L, 6, 6.0, new Date(),
+   *       7L, 7, 7.0, new Date(),
+   *       8L, 8888, 8.0, new Date(),
+   *       8L, 999, 9.0, new Date(),
+   *       10L, 100, 10.0, new Date())
+   * }</pre>
+   */
+  public static MockedBeamSQLTable of(final Object... args){
+    final RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        RelDataTypeFactory.FieldInfoBuilder builder = a0.builder();
+
+        int lastTypeIndex = 0;
+        for (; lastTypeIndex < args.length; lastTypeIndex += 2) {
+          if (args[lastTypeIndex] instanceof SqlTypeName) {
+            builder.add(args[lastTypeIndex + 1].toString(),
+                (SqlTypeName) args[lastTypeIndex]);
+          } else {
+            break;
+          }
+        }
+        return builder.build();
+      }
+    };
+
+    List<BeamSQLRow> rows = new ArrayList<>();
+    BeamSQLRecordType beamSQLRecordType = BeamSQLRecordType.from(
+        protoRowType.apply(BeamQueryPlanner.TYPE_FACTORY));
+    int fieldCount = beamSQLRecordType.size();
+
+    for (int i = fieldCount * 2; i < args.length; i += fieldCount) {
+      BeamSQLRow row = new BeamSQLRow(beamSQLRecordType);
+      for (int j = 0; j < fieldCount; j++) {
+        row.addField(j, args[i + j]);
+      }
+      rows.add(row);
+    }
+    return new MockedBeamSQLTable(protoRowType).withInputRecords(rows);
+  }
+
   @Override
   public BeamIOType getSourceType() {
     return BeamIOType.UNBOUNDED;
@@ -65,6 +128,10 @@ public PTransform<? super PCollection<BeamSQLRow>, PDone> buildIOWriter() {
     return new OutputStore();
   }
 
+  public List<BeamSQLRow> getInputRecords() {
+    return inputRecords;
+  }
+
   /**
    * Keep output in {@code CONTENT} for validation.
    *
@@ -93,7 +160,6 @@ public void close() {
       }));
       return PDone.in(input.getPipeline());
     }
-
   }
 
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
new file mode 100644
index 000000000000..11cec51845e5
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.rel;
+
+import java.util.Date;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
+import org.apache.beam.dsls.sql.planner.BeamSqlRunner;
+import org.apache.beam.dsls.sql.planner.MockedBeamSQLTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSortRel}.
+ */
+public class BeamSortRelTest {
+  public static BeamSqlRunner runner = new BeamSqlRunner();
+  private static MockedBeamSQLTable subOrderRamTable = MockedBeamSQLTable.of(
+      SqlTypeName.BIGINT, "order_id",
+      SqlTypeName.INTEGER, "site_id",
+      SqlTypeName.DOUBLE, "price");
+
+  private static MockedBeamSQLTable orderDetailTable = MockedBeamSQLTable
+      .of(SqlTypeName.BIGINT, "order_id",
+          SqlTypeName.INTEGER, "site_id",
+          SqlTypeName.DOUBLE, "price",
+          SqlTypeName.TIMESTAMP, "order_time",
+
+          1L, 2, 1.0, new Date(),
+          1L, 1, 2.0, new Date(),
+          2L, 4, 3.0, new Date(),
+          2L, 1, 4.0, new Date(),
+          5L, 5, 5.0, new Date(),
+          6L, 6, 6.0, new Date(),
+          7L, 7, 7.0, new Date(),
+          8L, 8888, 8.0, new Date(),
+          8L, 999, 9.0, new Date(),
+          10L, 100, 10.0, new Date());
+
+  @Test
+  public void testOrderBy_basic() throws Exception {
+    prepare();
+    String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
+        + " order_id, site_id, price "
+        + "FROM ORDER_DETAILS "
+        + "ORDER BY order_id asc, site_id desc limit 4";
+
+    System.out.println(sql);
+    runner.submitQuery(sql);
+
+    assertEquals(
+        MockedBeamSQLTable.of(
+            SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price",
+            1L, 2, 1.0,
+            1L, 1, 2.0,
+            2L, 4, 3.0,
+            2L, 1, 4.0
+        ).getInputRecords(), MockedBeamSQLTable.CONTENT);
+  }
+
+  @Test
+  public void testOrderBy_nullsFirst() throws Exception {
+    runner.addTable("ORDER_DETAILS", MockedBeamSQLTable
+        .of(SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price",
+
+            1L, 2, 1.0,
+            1L, null, 2.0,
+            2L, 1, 3.0,
+            2L, null, 4.0,
+            5L, 5, 5.0));
+    runner.addTable("SUB_ORDER_RAM", MockedBeamSQLTable
+        .of(SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price"));
+
+    String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
+        + " order_id, site_id, price "
+        + "FROM ORDER_DETAILS "
+        + "ORDER BY order_id asc, site_id desc NULLS FIRST limit 4";
+
+    runner.submitQuery(sql);
+
+    assertEquals(
+        MockedBeamSQLTable.of(
+            SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price",
+
+            1L, null, 2.0,
+            1L, 2, 1.0,
+            2L, null, 4.0,
+            2L, 1, 3.0
+        ).getInputRecords(), MockedBeamSQLTable.CONTENT);
+  }
+
+  @Test
+  public void testOrderBy_nullsLast() throws Exception {
+    runner.addTable("ORDER_DETAILS", MockedBeamSQLTable
+        .of(SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price",
+
+            1L, 2, 1.0,
+            1L, null, 2.0,
+            2L, 1, 3.0,
+            2L, null, 4.0,
+            5L, 5, 5.0));
+    runner.addTable("SUB_ORDER_RAM", MockedBeamSQLTable
+        .of(SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price"));
+
+    String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
+        + " order_id, site_id, price "
+        + "FROM ORDER_DETAILS "
+        + "ORDER BY order_id asc, site_id desc NULLS LAST limit 4";
+
+    runner.submitQuery(sql);
+
+    assertEquals(
+        MockedBeamSQLTable.of(
+            SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price",
+
+            1L, 2, 1.0,
+            1L, null, 2.0,
+            2L, 1, 3.0,
+            2L, null, 4.0
+        ).getInputRecords(), MockedBeamSQLTable.CONTENT);
+  }
+
+  @Test
+  public void testOrderBy_with_offset() throws Exception {
+    prepare();
+    String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
+        + " order_id, site_id, price "
+        + "FROM ORDER_DETAILS "
+        + "ORDER BY order_id asc, site_id desc limit 4 offset 4";
+
+    runner.submitQuery(sql);
+
+    assertEquals(
+        MockedBeamSQLTable.of(
+            SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price",
+
+            5L, 5, 5.0,
+            6L, 6, 6.0,
+            7L, 7, 7.0,
+            8L, 8888, 8.0
+        ).getInputRecords(), MockedBeamSQLTable.CONTENT);
+  }
+
+  @Test
+  public void testOrderBy_bigFetch() throws Exception {
+    prepare();
+    String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
+        + " order_id, site_id, price "
+        + "FROM ORDER_DETAILS "
+        + "ORDER BY order_id asc, site_id desc limit 11";
+
+    runner.submitQuery(sql);
+
+    assertEquals(
+        MockedBeamSQLTable.of(
+            SqlTypeName.BIGINT, "order_id",
+            SqlTypeName.INTEGER, "site_id",
+            SqlTypeName.DOUBLE, "price",
+
+            1L, 2, 1.0,
+            1L, 1, 2.0,
+            2L, 4, 3.0,
+            2L, 1, 4.0,
+            5L, 5, 5.0,
+            6L, 6, 6.0,
+            7L, 7, 7.0,
+            8L, 8888, 8.0,
+            8L, 999, 9.0,
+            10L, 100, 10.0
+        ).getInputRecords(), MockedBeamSQLTable.CONTENT);
+  }
+
+  @Test(expected = BeamSqlUnsupportedException.class)
+  public void testOrderBy_exception() throws Exception {
+    prepare();
+    String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id)  SELECT "
+        + " order_id, COUNT(*) "
+        + "FROM ORDER_DETAILS "
+        + "GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)"
+        + "ORDER BY order_id asc limit 11";
+
+    runner.submitQuery(sql);
+  }
+
+  public static void prepare() {
+    runner.addTable("ORDER_DETAILS", orderDetailTable);
+    runner.addTable("SUB_ORDER_RAM", subOrderRamTable);
+  }
+
+  private void assertEquals(List<BeamSQLRow> rows1, List<BeamSQLRow> rows2) {
+    Assert.assertEquals(rows1.size(), rows2.size());
+    for (int i = 0; i < rows1.size(); i++) {
+      Assert.assertEquals(rows1.get(i), rows2.get(i));
+    }
+  }
+}

From 44dd534e8d5345bc7e1ddc9b0f4bf011ec7bebe7 Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Sun, 14 May 2017 11:02:37 -0700
Subject: [PATCH 382/387] [BEAM-2288] Refine DSL interface as design doc of
 BEAM-2010: 1. rename BeamSqlRunner to BeamSQLEnvironment; 2. Move pipeline as
 external parameter, to integrate with none-DSL components;

---
 .../beam/dsls/sql/BeamSQLEnvironment.java     | 142 ++++++++++++++++++
 .../beam/dsls/sql/example/BeamSqlExample.java |  23 ++-
 .../dsls/sql/planner/BeamPipelineCreator.java |   9 +-
 .../dsls/sql/planner/BeamQueryPlanner.java    |  34 ++---
 .../beam/dsls/sql/planner/BeamSqlRunner.java  | 103 -------------
 .../beam/dsls/sql/planner/BasePlanner.java    |   9 +-
 .../sql/planner/BeamGroupByExplainTest.java   |  14 +-
 .../sql/planner/BeamGroupByPipelineTest.java  |  19 ++-
 .../sql/planner/BeamInvalidGroupByTest.java   |  10 +-
 .../BeamPlannerAggregationSubmitTest.java     |  17 ++-
 .../sql/planner/BeamPlannerExplainTest.java   |   7 +-
 .../sql/planner/BeamPlannerSubmitTest.java    |  11 +-
 .../beam/dsls/sql/rel/BeamSortRelTest.java    |  40 +++--
 13 files changed, 251 insertions(+), 187 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/BeamSQLEnvironment.java
 delete mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/BeamSQLEnvironment.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/BeamSQLEnvironment.java
new file mode 100644
index 000000000000..cdb25f50a45a
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/BeamSQLEnvironment.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql;
+
+import java.io.Serializable;
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.dsls.sql.rel.BeamRelNode;
+import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.impl.ScalarFunctionImpl;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@code BeamSQLEnvironment} is the integrated environment of BeamSQL.
+ * It provides runtime context to execute SQL queries as Beam pipeline,
+ * including table metadata, SQL engine and a Beam pipeline translator.
+ *
+ * <h1>1. BeamSQL as DSL</h1>
+ * <em>BeamSQL as DSL</em> enables developers to embed SQL queries when writing a Beam pipeline.
+ * A typical pipeline with BeamSQL DSL is:
+ * <pre>
+ *{@code
+PipelineOptions options =  PipelineOptionsFactory...
+Pipeline pipeline = Pipeline.create(options);
+
+//prepare environment of BeamSQL
+BeamSQLEnvironment sqlEnv = BeamSQLEnvironment.create();
+//register table metadata
+sqlEnv.addTableMetadata(String tableName, BeamSqlTable tableMetadata);
+//register UDF
+sqlEnv.registerUDF(String functionName, Method udfMethod);
+
+
+//explain a SQL statement, SELECT only, and return as a PCollection;
+PCollection<BeamSQLRow> phase1Stream = sqlEnv.explainSQL(pipeline, String sqlStatement);
+//A PCollection explained by BeamSQL can be converted into a table, and apply queries on it;
+sqlEnv.registerPCollectionAsTable(String tableName, phase1Stream);
+
+//apply more queries, even based on phase1Stream
+
+pipeline.run().waitUntilFinish();
+ * }
+ * </pre>
+ *
+ * <h1>2. BeamSQL as CLI</h1>
+ * This feature is on planning, and not ready yet.
+ *
+ */
+public class BeamSQLEnvironment implements Serializable {
+  private static final Logger LOG = LoggerFactory.getLogger(BeamSQLEnvironment.class);
+
+  public static final BeamSQLEnvironment INSTANCE = new BeamSQLEnvironment();
+
+  private SchemaPlus schema = Frameworks.createRootSchema(true);
+  private BeamQueryPlanner planner = new BeamQueryPlanner(schema);
+
+  private BeamSQLEnvironment() {
+    //disable assertions in Calcite.
+    ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(false);
+  }
+
+  /**
+   * Return an instance of {@code BeamSQLEnvironment}.
+   */
+  public static BeamSQLEnvironment create(){
+    return INSTANCE;
+  }
+
+  /**
+   * Add a schema.
+   *
+   */
+  public void addSchema(String schemaName, Schema scheme) {
+    schema.add(schemaName, schema);
+  }
+
+  /**
+   * add a {@link BaseBeamTable} to schema repository.
+   */
+  public void addTableMetadata(String tableName, BaseBeamTable tableMetadata) {
+    schema.add(tableName, tableMetadata);
+    planner.getSourceTables().put(tableName, tableMetadata);
+  }
+
+  /* Add a UDF function.
+   *
+   * <p>There're two requirements for function {@code methodName}:<br>
+   * 1. It must be a STATIC method;<br>
+   * 2. For a primitive parameter, use its wrapper class and handle NULL properly;
+   */
+  public void addUDFFunction(String functionName, Class<?> className, String methodName){
+    schema.add(functionName, ScalarFunctionImpl.create(className, methodName));
+  }
+
+  /**
+   * explain and display the execution plan.
+   */
+  public String executionPlan(String sqlString)
+    throws ValidationException, RelConversionException, SqlParseException {
+    BeamRelNode exeTree = planner.convertToBeamRel(sqlString);
+    String beamPlan = RelOptUtil.toString(exeTree);
+    LOG.info(String.format("beamPlan>\n%s", beamPlan));
+    return beamPlan;
+  }
+
+  /**
+   * {@code compileBeamPipeline} translate a SQL statement to executed as Beam data flow,
+   * which is linked with the given {@code pipeline}. The final output stream is returned as
+   * {@code PCollection} so more operations can be applied.
+   */
+  public PCollection<BeamSQLRow> compileBeamPipeline(String sqlStatement, Pipeline basePipeline)
+      throws Exception{
+    PCollection<BeamSQLRow> resultStream = planner.compileBeamPipeline(sqlStatement, basePipeline);
+    return resultStream;
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
index 303835fd6d0a..26959446f167 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
@@ -21,9 +21,14 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.beam.dsls.sql.planner.BeamSqlRunner;
+import org.apache.beam.dsls.sql.BeamSQLEnvironment;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.dsls.sql.schema.kafka.BeamKafkaCSVTable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelProtoDataType;
@@ -67,17 +72,23 @@
 public class BeamSqlExample implements Serializable {
 
   public static void main(String[] args) throws Exception {
-    BeamSqlRunner runner = new BeamSqlRunner();
-    runner.addTable("ORDER_DETAILS", getTable("127.0.0.1:9092", "orders"));
-    runner.addTable("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
+    PipelineOptions options = PipelineOptionsFactory.fromArgs(new String[] {}).withValidation()
+        .as(PipelineOptions.class); // FlinkPipelineOptions.class
+    options.setJobName("BeamSqlExample");
+    Pipeline pipeline = Pipeline.create(options);
+
+    BeamSQLEnvironment runner = BeamSQLEnvironment.create();
+    runner.addTableMetadata("ORDER_DETAILS", getTable("127.0.0.1:9092", "orders"));
+    runner.addTableMetadata("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
 
     // case 2: insert into <table>(<fields>) select STREAM <fields> from
     // <table> from <clause>
     String sql = "INSERT INTO SUB_ORDER(order_id, site_id, price) " + "SELECT "
         + " order_id, site_id, price " + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 and price > 20";
 
-    runner.explainQuery(sql);
-    runner.submitQuery(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
+
+    pipeline.run().waitUntilFinish();
   }
 
   public static BaseBeamTable getTable(String bootstrapServer, String topic) {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
index 98ccb57e939b..1f3ba58de2df 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
@@ -28,7 +28,6 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.CoderRegistry;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
 
 /**
  * {@link BeamPipelineCreator} converts a {@link BeamRelNode} tree, into a Beam
@@ -44,14 +43,10 @@ public class BeamPipelineCreator {
 
   private boolean hasPersistent = false;
 
-  public BeamPipelineCreator(Map<String, BaseBeamTable> sourceTables) {
+  public BeamPipelineCreator(Map<String, BaseBeamTable> sourceTables, Pipeline pipeline) {
     this.sourceTables = sourceTables;
+    this.pipeline = pipeline;
 
-    options = PipelineOptionsFactory.fromArgs(new String[] {}).withValidation()
-        .as(PipelineOptions.class); // FlinkPipelineOptions.class
-    options.setJobName("BeamPlanCreator");
-
-    pipeline = Pipeline.create(options);
     CoderRegistry cr = pipeline.getCoderRegistry();
     cr.registerCoder(BeamSQLRow.class, BeamSqlRowCoder.of());
     cr.registerCoder(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
index 9e4155592bff..0a7407c3e5c2 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamQueryPlanner.java
@@ -26,9 +26,9 @@
 import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
 import org.apache.beam.dsls.sql.rel.BeamRelNode;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
-
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.adapter.java.JavaTypeFactory;
 import org.apache.calcite.config.Lex;
 import org.apache.calcite.jdbc.CalciteSchema;
@@ -58,8 +58,8 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * The core component to handle through a SQL statement, to submit a Beam
- * pipeline.
+ * The core component to handle through a SQL statement, from explain execution plan,
+ * to generate a Beam pipeline.
  *
  */
 public class BeamQueryPlanner {
@@ -95,29 +95,17 @@ public BeamQueryPlanner(SchemaPlus schema) {
   }
 
   /**
-   * With a Beam pipeline generated in {@link #compileBeamPipeline(String)},
-   * submit it to run and wait until finish.
-   *
-   */
-  public void submitToRun(String sqlStatement) throws Exception {
-    Pipeline pipeline = compileBeamPipeline(sqlStatement);
-
-    PipelineResult result = pipeline.run();
-    result.waitUntilFinish();
-  }
-
-  /**
-   * With the @{@link BeamRelNode} tree generated in
-   * {@link #convertToBeamRel(String)}, a Beam pipeline is generated.
-   *
+   * {@code compileBeamPipeline} translate a SQL statement to executed as Beam data flow,
+   * which is linked with the given {@code pipeline}. The final output stream is returned as
+   * {@code PCollection} so more operations can be applied.
    */
-  public Pipeline compileBeamPipeline(String sqlStatement) throws Exception {
+  public PCollection<BeamSQLRow> compileBeamPipeline(String sqlStatement, Pipeline pipeline)
+      throws Exception {
     BeamRelNode relNode = convertToBeamRel(sqlStatement);
 
-    BeamPipelineCreator planCreator = new BeamPipelineCreator(sourceTables);
-    relNode.buildBeamPipeline(planCreator);
+    BeamPipelineCreator planCreator = new BeamPipelineCreator(sourceTables, pipeline);
 
-    return planCreator.getPipeline();
+    return relNode.buildBeamPipeline(planCreator);
   }
 
   /**
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
deleted file mode 100644
index 95ba5a959f7f..000000000000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRunner.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.planner;
-
-import java.io.Serializable;
-
-import org.apache.beam.dsls.sql.rel.BeamRelNode;
-import org.apache.beam.dsls.sql.schema.BaseBeamTable;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.schema.Schema;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.impl.ScalarFunctionImpl;
-import org.apache.calcite.sql.parser.SqlParseException;
-import org.apache.calcite.tools.Frameworks;
-import org.apache.calcite.tools.RelConversionException;
-import org.apache.calcite.tools.ValidationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Interface to explain, submit a SQL query.
- *
- */
-public class BeamSqlRunner implements Serializable {
-  private static final Logger LOG = LoggerFactory.getLogger(BeamSqlRunner.class);
-
-  private SchemaPlus schema = Frameworks.createRootSchema(true);
-
-  private BeamQueryPlanner planner = new BeamQueryPlanner(schema);
-
-  public BeamSqlRunner() {
-    //disable assertions in Calcite.
-    ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(false);
-  }
-
-  /**
-   * Add a schema.
-   *
-   */
-  public void addSchema(String schemaName, Schema scheme) {
-    schema.add(schemaName, schema);
-  }
-
-  /**
-   * add a {@link BaseBeamTable} to schema repository.
-   *
-   */
-  public void addTable(String tableName, BaseBeamTable table) {
-    schema.add(tableName, table);
-    planner.getSourceTables().put(tableName, table);
-  }
-
-  /**
-   * Add a UDF function.
-   *
-   * <p>There're two requirements for function {@code methodName}:<br>
-   * 1. It must be a STATIC method;<br>
-   * 2. For a primitive parameter, use its wrapper class and handle NULL properly;
-   */
-  public void addUDFFunction(String functionName, Class<?> className, String methodName){
-    schema.add(functionName, ScalarFunctionImpl.create(className, methodName));
-  }
-
-  /**
-   * submit as a Beam pipeline.
-   *
-   */
-  public void submitQuery(String sqlString) throws Exception {
-    planner.submitToRun(sqlString);
-  }
-
-  /**
-   * explain and display the execution plan.
-   *
-   */
-  public String explainQuery(String sqlString)
-    throws ValidationException, RelConversionException, SqlParseException {
-    BeamRelNode exeTree = planner.convertToBeamRel(sqlString);
-    String beamPlan = RelOptUtil.toString(exeTree);
-    System.out.println(String.format("beamPlan>\n%s", beamPlan));
-    return beamPlan;
-  }
-
-  protected BeamQueryPlanner getPlanner() {
-    return planner;
-  }
-
-}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
index 055a4d4e8e6c..fe8a2364bb84 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
@@ -21,6 +21,7 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.beam.dsls.sql.BeamSQLEnvironment;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
@@ -37,13 +38,13 @@
  *
  */
 public class BasePlanner {
-  public static BeamSqlRunner runner = new BeamSqlRunner();
+  public static BeamSQLEnvironment runner = BeamSQLEnvironment.create();
 
   @BeforeClass
   public static void prepare() {
-    runner.addTable("ORDER_DETAILS", getTable());
-    runner.addTable("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
-    runner.addTable("SUB_ORDER_RAM", getTable());
+    runner.addTableMetadata("ORDER_DETAILS", getTable());
+    runner.addTableMetadata("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
+    runner.addTableMetadata("SUB_ORDER_RAM", getTable());
   }
 
   private static BaseBeamTable getTable() {
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
index 4f2b1ba91407..98d14c305cec 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByExplainTest.java
@@ -33,7 +33,7 @@ public class BeamGroupByExplainTest extends BasePlanner {
   public void testSimpleGroupExplain() throws Exception {
     String sql = "SELECT COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 ";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
   }
 
   /**
@@ -43,7 +43,7 @@ public void testSimpleGroupExplain() throws Exception {
   public void testSimpleGroup2Explain() throws Exception {
     String sql = "SELECT site_id" + ", COUNT(*) " + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY site_id";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
   }
 
   /**
@@ -54,7 +54,7 @@ public void testTumbleExplain() throws Exception {
     String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
         + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
   }
 
   /**
@@ -66,7 +66,7 @@ public void testTumbleWithDelayExplain() throws Exception {
         + "TUMBLE_START(order_time, INTERVAL '1' HOUR, TIME '00:00:01')"
         + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 "
         + "GROUP BY order_id, site_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR, TIME '00:00:01')";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
   }
 
   /**
@@ -77,7 +77,7 @@ public void testHopExplain() throws Exception {
     String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
         + ", HOP(order_time, INTERVAL '5' MINUTE, INTERVAL '1' HOUR)";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
   }
 
   /**
@@ -88,7 +88,7 @@ public void testSessionExplain() throws Exception {
     String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
         + ", SESSION(order_time, INTERVAL '5' MINUTE)";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
   }
 
   /**
@@ -99,6 +99,6 @@ public void testUdf() throws Exception {
     runner.addUDFFunction("negative", BeamSqlUdfExpressionTest.UdfFn.class, "negative");
     String sql = "select site_id, negative(site_id) as nsite_id from ORDER_DETAILS";
 
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
   }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
index 71dcf73d6b71..5101c9809eb5 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamGroupByPipelineTest.java
@@ -18,7 +18,9 @@
 package org.apache.beam.dsls.sql.planner;
 
 import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlUdfExpressionTest;
-import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
 import org.junit.Test;
 
 /**
@@ -26,6 +28,7 @@
  *
  */
 public class BeamGroupByPipelineTest extends BasePlanner {
+  public final TestPipeline pipeline = TestPipeline.create();
 
   /**
    * GROUP-BY without window operation, and grouped fields.
@@ -34,7 +37,7 @@ public class BeamGroupByPipelineTest extends BasePlanner {
   public void testSimpleGroupExplain() throws Exception {
     String sql = "SELECT COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 ";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
   /**
@@ -44,7 +47,7 @@ public void testSimpleGroupExplain() throws Exception {
   public void testSimpleGroup2Explain() throws Exception {
     String sql = "SELECT site_id" + ", COUNT(*) " + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY site_id";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
   /**
@@ -55,7 +58,7 @@ public void testTumbleExplain() throws Exception {
     String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
         + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
   /**
@@ -67,7 +70,7 @@ public void testTumbleWithDelayExplain() throws Exception {
         + "TUMBLE_START(order_time, INTERVAL '1' HOUR, TIME '00:00:01')"
         + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 "
         + "GROUP BY order_id, site_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR, TIME '00:00:01')";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
   /**
@@ -78,7 +81,7 @@ public void testHopExplain() throws Exception {
     String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
         + ", HOP(order_time, INTERVAL '5' MINUTE, INTERVAL '1' HOUR)";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
   /**
@@ -89,7 +92,7 @@ public void testSessionExplain() throws Exception {
     String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY order_id, site_id"
         + ", SESSION(order_time, INTERVAL '5' MINUTE)";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
   /**
@@ -100,7 +103,7 @@ public void testUdf() throws Exception {
     runner.addUDFFunction("negative", BeamSqlUdfExpressionTest.UdfFn.class, "negative");
     String sql = "select site_id, negative(site_id) as nsite_id from ORDER_DETAILS";
 
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java
index 83ab8716f6f8..72b5bf76460c 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamInvalidGroupByTest.java
@@ -17,7 +17,11 @@
  */
 package org.apache.beam.dsls.sql.planner;
 
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
 import org.apache.calcite.tools.ValidationException;
+import org.junit.Rule;
 import org.junit.Test;
 
 /**
@@ -25,12 +29,14 @@
  *
  */
 public class BeamInvalidGroupByTest extends BasePlanner {
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
 
   @Test(expected = ValidationException.class)
   public void testTumble2Explain() throws Exception {
     String sql = "SELECT order_id, site_id" + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY order_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
-    String plan = runner.explainQuery(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
   @Test(expected = ValidationException.class)
@@ -38,7 +44,7 @@ public void testTumble3Explain() throws Exception {
     String sql = "SELECT order_id, site_id, TUMBLE(order_time, INTERVAL '1' HOUR)"
         + ", COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 "
         + "GROUP BY order_id, site_id" + ", TUMBLE(order_time, INTERVAL '1' HOUR)";
-    String plan = runner.explainQuery(sql);
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
   }
 
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
index 39a56143b27d..22f1848deb46 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
@@ -21,16 +21,18 @@
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
+import org.apache.beam.dsls.sql.BeamSQLEnvironment;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
-import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelProtoDataType;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.junit.Assert;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
 
 /**
@@ -39,12 +41,15 @@
  */
 public class BeamPlannerAggregationSubmitTest {
   public static DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-  public static BeamSqlRunner runner = new BeamSqlRunner();
+  public static BeamSQLEnvironment runner = BeamSQLEnvironment.create();
+
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
 
   @BeforeClass
   public static void prepare() throws ParseException {
-    runner.addTable("ORDER_DETAILS", getOrderTable());
-    runner.addTable("ORDER_SUMMARY", getSummaryTable());
+    runner.addTableMetadata("ORDER_DETAILS", getOrderTable());
+    runner.addTableMetadata("ORDER_SUMMARY", getSummaryTable());
   }
 
   private static BaseBeamTable getOrderTable() throws ParseException {
@@ -108,7 +113,7 @@ public void selectWithWindowAggregation() throws Exception{
         + "WHERE SITE_ID = 1 " + "GROUP BY site_id"
         + ", TUMBLE(order_time, INTERVAL '1' HOUR, TIME '00:00:01')";
 
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    runner.compileBeamPipeline(sql, pipeline);
 
     pipeline.run().waitUntilFinish();
 
@@ -125,7 +130,7 @@ public void selectWithoutWindowAggregation() throws Exception{
         + "SELECT site_id, COUNT(*) AS `SIZE`" + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 " + "GROUP BY site_id";
 
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+    runner.compileBeamPipeline(sql, pipeline);
 
     pipeline.run().waitUntilFinish();
 
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
index 9b6b20a1743c..1355d5d27a3b 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerExplainTest.java
@@ -25,11 +25,10 @@
  *
  */
 public class BeamPlannerExplainTest extends BasePlanner {
-
   @Test
   public void selectAll() throws Exception {
     String sql = "SELECT * FROM ORDER_DETAILS";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
 
     String expectedPlan =
         "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2], order_time=[$3])\n"
@@ -41,7 +40,7 @@ public void selectAll() throws Exception {
   public void selectWithFilter() throws Exception {
     String sql = "SELECT " + " order_id, site_id, price " + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 and price > 20";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
 
     String expectedPlan = "BeamProjectRel(order_id=[$0], site_id=[$1], price=[$2])\n"
         + "  BeamFilterRel(condition=[AND(=($1, 0), >($2, 20))])\n"
@@ -54,7 +53,7 @@ public void insertSelectFilter() throws Exception {
     String sql = "INSERT INTO SUB_ORDER(order_id, site_id, price) " + "SELECT "
         + " order_id, site_id, price " + "FROM ORDER_DETAILS "
         + "WHERE SITE_ID = 0 and price > 20";
-    String plan = runner.explainQuery(sql);
+    String plan = runner.executionPlan(sql);
 
     String expectedPlan =
         "BeamIOSinkRel(table=[[SUB_ORDER]], operation=[INSERT], flattened=[true])\n"
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
index 54350491c996..17cea274a9dc 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
@@ -17,8 +17,11 @@
  */
 package org.apache.beam.dsls.sql.planner;
 
-import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
 import org.junit.Assert;
+import org.junit.Rule;
 import org.junit.Test;
 
 /**
@@ -26,12 +29,16 @@
  *
  */
 public class BeamPlannerSubmitTest extends BasePlanner {
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
+
   @Test
   public void insertSelectFilter() throws Exception {
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
         + " order_id, site_id, price "
         + "FROM ORDER_DETAILS " + "WHERE SITE_ID = 0 and price > 20";
-    Pipeline pipeline = runner.getPlanner().compileBeamPipeline(sql);
+
+    PCollection<BeamSQLRow> outputStream = runner.compileBeamPipeline(sql, pipeline);
 
     pipeline.run().waitUntilFinish();
 
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
index 11cec51845e5..864d4b7c97fe 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
@@ -20,20 +20,24 @@
 
 import java.util.Date;
 import java.util.List;
-
+import org.apache.beam.dsls.sql.BeamSQLEnvironment;
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
-import org.apache.beam.dsls.sql.planner.BeamSqlRunner;
 import org.apache.beam.dsls.sql.planner.MockedBeamSQLTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.junit.Assert;
+import org.junit.Rule;
 import org.junit.Test;
 
 /**
  * Test for {@code BeamSortRel}.
  */
 public class BeamSortRelTest {
-  public static BeamSqlRunner runner = new BeamSqlRunner();
+  public static BeamSQLEnvironment runner = BeamSQLEnvironment.create();
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
+
   private static MockedBeamSQLTable subOrderRamTable = MockedBeamSQLTable.of(
       SqlTypeName.BIGINT, "order_id",
       SqlTypeName.INTEGER, "site_id",
@@ -65,7 +69,8 @@ public void testOrderBy_basic() throws Exception {
         + "ORDER BY order_id asc, site_id desc limit 4";
 
     System.out.println(sql);
-    runner.submitQuery(sql);
+    runner.compileBeamPipeline(sql, pipeline);
+    pipeline.run().waitUntilFinish();
 
     assertEquals(
         MockedBeamSQLTable.of(
@@ -81,7 +86,7 @@ public void testOrderBy_basic() throws Exception {
 
   @Test
   public void testOrderBy_nullsFirst() throws Exception {
-    runner.addTable("ORDER_DETAILS", MockedBeamSQLTable
+    runner.addTableMetadata("ORDER_DETAILS", MockedBeamSQLTable
         .of(SqlTypeName.BIGINT, "order_id",
             SqlTypeName.INTEGER, "site_id",
             SqlTypeName.DOUBLE, "price",
@@ -91,7 +96,7 @@ public void testOrderBy_nullsFirst() throws Exception {
             2L, 1, 3.0,
             2L, null, 4.0,
             5L, 5, 5.0));
-    runner.addTable("SUB_ORDER_RAM", MockedBeamSQLTable
+    runner.addTableMetadata("SUB_ORDER_RAM", MockedBeamSQLTable
         .of(SqlTypeName.BIGINT, "order_id",
             SqlTypeName.INTEGER, "site_id",
             SqlTypeName.DOUBLE, "price"));
@@ -101,7 +106,8 @@ public void testOrderBy_nullsFirst() throws Exception {
         + "FROM ORDER_DETAILS "
         + "ORDER BY order_id asc, site_id desc NULLS FIRST limit 4";
 
-    runner.submitQuery(sql);
+    runner.compileBeamPipeline(sql, pipeline);
+    pipeline.run().waitUntilFinish();
 
     assertEquals(
         MockedBeamSQLTable.of(
@@ -118,7 +124,7 @@ public void testOrderBy_nullsFirst() throws Exception {
 
   @Test
   public void testOrderBy_nullsLast() throws Exception {
-    runner.addTable("ORDER_DETAILS", MockedBeamSQLTable
+    runner.addTableMetadata("ORDER_DETAILS", MockedBeamSQLTable
         .of(SqlTypeName.BIGINT, "order_id",
             SqlTypeName.INTEGER, "site_id",
             SqlTypeName.DOUBLE, "price",
@@ -128,7 +134,7 @@ public void testOrderBy_nullsLast() throws Exception {
             2L, 1, 3.0,
             2L, null, 4.0,
             5L, 5, 5.0));
-    runner.addTable("SUB_ORDER_RAM", MockedBeamSQLTable
+    runner.addTableMetadata("SUB_ORDER_RAM", MockedBeamSQLTable
         .of(SqlTypeName.BIGINT, "order_id",
             SqlTypeName.INTEGER, "site_id",
             SqlTypeName.DOUBLE, "price"));
@@ -138,7 +144,8 @@ public void testOrderBy_nullsLast() throws Exception {
         + "FROM ORDER_DETAILS "
         + "ORDER BY order_id asc, site_id desc NULLS LAST limit 4";
 
-    runner.submitQuery(sql);
+    runner.compileBeamPipeline(sql, pipeline);
+    pipeline.run().waitUntilFinish();
 
     assertEquals(
         MockedBeamSQLTable.of(
@@ -161,7 +168,8 @@ public void testOrderBy_with_offset() throws Exception {
         + "FROM ORDER_DETAILS "
         + "ORDER BY order_id asc, site_id desc limit 4 offset 4";
 
-    runner.submitQuery(sql);
+    runner.compileBeamPipeline(sql, pipeline);
+    pipeline.run().waitUntilFinish();
 
     assertEquals(
         MockedBeamSQLTable.of(
@@ -184,7 +192,8 @@ public void testOrderBy_bigFetch() throws Exception {
         + "FROM ORDER_DETAILS "
         + "ORDER BY order_id asc, site_id desc limit 11";
 
-    runner.submitQuery(sql);
+    runner.compileBeamPipeline(sql, pipeline);
+    pipeline.run().waitUntilFinish();
 
     assertEquals(
         MockedBeamSQLTable.of(
@@ -214,12 +223,13 @@ public void testOrderBy_exception() throws Exception {
         + "GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)"
         + "ORDER BY order_id asc limit 11";
 
-    runner.submitQuery(sql);
+    TestPipeline pipeline = TestPipeline.create();
+    runner.compileBeamPipeline(sql, pipeline);
   }
 
   public static void prepare() {
-    runner.addTable("ORDER_DETAILS", orderDetailTable);
-    runner.addTable("SUB_ORDER_RAM", subOrderRamTable);
+    runner.addTableMetadata("ORDER_DETAILS", orderDetailTable);
+    runner.addTableMetadata("SUB_ORDER_RAM", subOrderRamTable);
   }
 
   private void assertEquals(List<BeamSQLRow> rows1, List<BeamSQLRow> rows2) {

From 1340a9f8415d52bd6941693ade4c62142e3de85d Mon Sep 17 00:00:00 2001
From: mingmxu <mingmxu@ebay.com>
Date: Sun, 14 May 2017 13:22:30 -0700
Subject: [PATCH 383/387] [BEAM-2292] Add BeamPCollectionTable to create table
 from PCollection<BeamSQLRow>

---
 .../beam/dsls/sql/rel/BeamIOSourceRel.java    |  3 +-
 .../beam/dsls/sql/schema/BaseBeamTable.java   |  6 +-
 .../dsls/sql/schema/BeamPCollectionTable.java | 62 ++++++++++++++++++
 .../dsls/sql/schema/kafka/BeamKafkaTable.java | 12 +---
 .../sql/schema/text/BeamTextCSVTable.java     |  8 ++-
 .../schema/text/BeamTextCSVTableIOReader.java |  9 +--
 .../dsls/sql/planner/MockedBeamSQLTable.java  |  5 +-
 .../sql/schema/BeamPCollectionTableTest.java  | 64 +++++++++++++++++++
 .../sql/schema/text/BeamTextCSVTableTest.java | 12 ++--
 9 files changed, 150 insertions(+), 31 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTable.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTableTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
index 61f53eb14893..f4d5001cc90e 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java
@@ -49,8 +49,7 @@ public PCollection<BeamSQLRow> buildBeamPipeline(BeamPipelineCreator planCreator
 
     String stageName = BeamSQLRelUtils.getStageName(this);
 
-    PCollection<BeamSQLRow> sourceStream = planCreator.getPipeline().apply(stageName,
-        sourceTable.buildIOReader());
+    PCollection<BeamSQLRow> sourceStream = sourceTable.buildIOReader(planCreator.getPipeline());
 
     return sourceStream;
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
index 2ecfa38f70cd..52d2bbdb7e52 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BaseBeamTable.java
@@ -19,8 +19,8 @@
 
 import java.io.Serializable;
 import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.calcite.DataContext;
@@ -53,10 +53,10 @@ public BaseBeamTable(RelProtoDataType protoRowType) {
   public abstract BeamIOType getSourceType();
 
   /**
-   * create a {@code IO.read()} instance to read from source.
+   * create a {@code PCollection<BeamSQLRow>} from source.
    *
    */
-  public abstract PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader();
+  public abstract PCollection<BeamSQLRow> buildIOReader(Pipeline pipeline);
 
   /**
    * create a {@code IO.write()} instance to write to target.
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTable.java
new file mode 100644
index 000000000000..1c3ab5b77ebe
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTable.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.schema;
+
+import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.calcite.rel.type.RelProtoDataType;
+
+/**
+ * {@code BeamPCollectionTable} converts a {@code PCollection<BeamSQLRow>} as a virtual table,
+ * then a downstream query can query directly.
+ */
+public class BeamPCollectionTable extends BaseBeamTable {
+  private BeamIOType ioType;
+  private PCollection<BeamSQLRow> upstream;
+
+  protected BeamPCollectionTable(RelProtoDataType protoRowType) {
+    super(protoRowType);
+  }
+
+  public BeamPCollectionTable(PCollection<BeamSQLRow> upstream, RelProtoDataType protoRowType){
+    this(protoRowType);
+    ioType = upstream.isBounded().equals(IsBounded.BOUNDED)
+        ? BeamIOType.BOUNDED : BeamIOType.UNBOUNDED;
+    this.upstream = upstream;
+  }
+
+  @Override
+  public BeamIOType getSourceType() {
+    return ioType;
+  }
+
+  @Override
+  public PCollection<BeamSQLRow> buildIOReader(Pipeline pipeline) {
+    return upstream;
+  }
+
+  @Override
+  public PTransform<? super PCollection<BeamSQLRow>, PDone> buildIOWriter() {
+    throw new BeamInvalidOperatorException("cannot use [BeamPCollectionTable] as target");
+  }
+
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
index c8c851c17d2a..7342cee38714 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
@@ -24,6 +24,7 @@
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamIOType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.io.kafka.KafkaIO;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -72,19 +73,12 @@ public BeamIOType getSourceType() {
       getPTransformForOutput();
 
   @Override
-  public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
-    return new PTransform<PBegin, PCollection<BeamSQLRow>>() {
-
-      @Override
-      public PCollection<BeamSQLRow> expand(PBegin input) {
-        return input.apply("read",
+  public PCollection<BeamSQLRow> buildIOReader(Pipeline pipeline) {
+    return PBegin.in(pipeline).apply("read",
             KafkaIO.<byte[], byte[]>read().withBootstrapServers(bootstrapServers).withTopics(topics)
                 .updateConsumerProperties(configUpdates).withKeyCoder(ByteArrayCoder.of())
                 .withValueCoder(ByteArrayCoder.of()).withoutMetadata())
             .apply("in_format", getPTransformForInput());
-
-      }
-    };
   }
 
   @Override
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
index b9e6b816529f..6b2128945b16 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
@@ -19,6 +19,8 @@
 package org.apache.beam.dsls.sql.schema.text;
 
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -55,8 +57,10 @@ public BeamTextCSVTable(RelProtoDataType protoDataType, String filePattern,
   }
 
   @Override
-  public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
-    return new BeamTextCSVTableIOReader(beamSqlRecordType, filePattern, csvFormat);
+  public PCollection<BeamSQLRow> buildIOReader(Pipeline pipeline) {
+    return PBegin.in(pipeline).apply("decodeRecord", TextIO.Read.from(filePattern))
+        .apply("parseCSVLine",
+            new BeamTextCSVTableIOReader(beamSqlRecordType, filePattern, csvFormat));
   }
 
   @Override
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
index 3c031cead6b8..59d77c07a485 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOReader.java
@@ -24,11 +24,9 @@
 
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
-import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.commons.csv.CSVFormat;
 
@@ -36,7 +34,7 @@
  * IOReader for {@code BeamTextCSVTable}.
  */
 public class BeamTextCSVTableIOReader
-    extends PTransform<PBegin, PCollection<BeamSQLRow>>
+    extends PTransform<PCollection<String>, PCollection<BeamSQLRow>>
     implements Serializable {
   private String filePattern;
   protected BeamSQLRecordType beamSqlRecordType;
@@ -50,9 +48,8 @@ public BeamTextCSVTableIOReader(BeamSQLRecordType beamSqlRecordType, String file
   }
 
   @Override
-  public PCollection<BeamSQLRow> expand(PBegin input) {
-    return input.apply("decodeRecord", TextIO.Read.from(filePattern))
-        .apply(ParDo.of(new DoFn<String, BeamSQLRow>() {
+  public PCollection<BeamSQLRow> expand(PCollection<String> input) {
+    return input.apply(ParDo.of(new DoFn<String, BeamSQLRow>() {
           @ProcessElement
           public void processElement(ProcessContext ctx) {
             String str = ctx.element();
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
index 8ccb33212f9f..78fd05534a52 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
@@ -24,6 +24,7 @@
 import org.apache.beam.dsls.sql.schema.BeamIOType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -119,8 +120,8 @@ public BeamIOType getSourceType() {
   }
 
   @Override
-  public PTransform<? super PBegin, PCollection<BeamSQLRow>> buildIOReader() {
-    return Create.of(inputRecords);
+  public PCollection<BeamSQLRow> buildIOReader(Pipeline pipeline) {
+    return PBegin.in(pipeline).apply(Create.of(inputRecords));
   }
 
   @Override
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTableTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTableTest.java
new file mode 100644
index 000000000000..6f24e2ab9a00
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamPCollectionTableTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.schema;
+
+import org.apache.beam.dsls.sql.planner.BasePlanner;
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test case for BeamPCollectionTable.
+ */
+public class BeamPCollectionTableTest extends BasePlanner{
+  public static TestPipeline pipeline = TestPipeline.create();
+
+  @BeforeClass
+  public static void prepareTable(){
+    RelProtoDataType protoRowType = new RelProtoDataType() {
+      @Override
+      public RelDataType apply(RelDataTypeFactory a0) {
+        return a0.builder().add("c1", SqlTypeName.INTEGER)
+            .add("c2", SqlTypeName.VARCHAR).build();
+      }
+    };
+
+    BeamSQLRow row = new BeamSQLRow(BeamSQLRecordType.from(
+        protoRowType.apply(BeamQueryPlanner.TYPE_FACTORY)));
+    row.addField(0, 1);
+    row.addField(1, "hello world.");
+    PCollection<BeamSQLRow> inputStream = PBegin.in(pipeline).apply(Create.of(row));
+    runner.addTableMetadata("COLLECTION_TABLE",
+        new BeamPCollectionTable(inputStream, protoRowType));
+  }
+
+  @Test
+  public void testSelectFromPCollectionTable() throws Exception{
+    String sql = "select c1, c2 from COLLECTION_TABLE";
+    runner.executionPlan(sql);
+  }
+
+}
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
index 3bc29e41ff20..4c403ac26c2f 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
@@ -80,22 +80,20 @@ public class BeamTextCSVTableTest {
   private static File writerTargetFile;
 
   @Test public void testBuildIOReader() {
-    PCollection<BeamSQLRow> rows = pipeline.apply(
-        new BeamTextCSVTable(buildRowType(), readerSourceFile.getAbsolutePath()).buildIOReader());
+    PCollection<BeamSQLRow> rows = new BeamTextCSVTable(buildRowType(),
+        readerSourceFile.getAbsolutePath()).buildIOReader(pipeline);
     PAssert.that(rows).containsInAnyOrder(testDataRows);
     pipeline.run();
   }
 
   @Test public void testBuildIOWriter() {
-    // reader from a source file, then write into a target file
-    pipeline.apply(
-        new BeamTextCSVTable(buildRowType(), readerSourceFile.getAbsolutePath()).buildIOReader())
+    new BeamTextCSVTable(buildRowType(), readerSourceFile.getAbsolutePath()).buildIOReader(pipeline)
         .apply(new BeamTextCSVTable(buildRowType(), writerTargetFile.getAbsolutePath())
             .buildIOWriter());
     pipeline.run();
 
-    PCollection<BeamSQLRow> rows = pipeline2.apply(
-        new BeamTextCSVTable(buildRowType(), writerTargetFile.getAbsolutePath()).buildIOReader());
+    PCollection<BeamSQLRow> rows = new BeamTextCSVTable(buildRowType(),
+        writerTargetFile.getAbsolutePath()).buildIOReader(pipeline2);
 
     // confirm the two reads match
     PAssert.that(rows).containsInAnyOrder(testDataRows);

From 0308c2725faa507f2e0235aca5b11f4c31a5a9b0 Mon Sep 17 00:00:00 2001
From: unknown <冯飞>
Date: Mon, 15 May 2017 03:02:48 +0100
Subject: [PATCH 384/387] fix NoSuchFieldException

---
 .../org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
index eb9fedf58842..be388aacbac1 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -108,7 +108,7 @@ static BeamSqlExpression buildExpression(RexNode rexNode) {
       RexCall node = (RexCall) rexNode;
       String opName = node.op.getName();
       List<BeamSqlExpression> subExps = new ArrayList<>();
-      for (RexNode subNode : node.operands) {
+      for (RexNode subNode : node.getOperands()) {
         subExps.add(buildExpression(subNode));
       }
       switch (opName) {

From 5d9707cb7e5ad27228a5871d9141a56e32a297d4 Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Wed, 17 May 2017 14:41:18 +0800
Subject: [PATCH 385/387] [BEAM-2309] Implement VALUES and add support for data
 type CHAR (to be able to test VALUES)

---
 .../sql/interpreter/BeamSQLFnExecutor.java    | 12 ++-
 .../operator/BeamSqlPrimitive.java            |  1 +
 .../beam/dsls/sql/planner/BeamRuleSets.java   |  3 +-
 .../beam/dsls/sql/rel/BeamValuesRel.java      | 78 +++++++++++++++
 .../beam/dsls/sql/rule/BeamValuesRule.java    | 48 ++++++++++
 .../beam/dsls/sql/schema/BeamSQLRow.java      |  2 +
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java |  2 +
 .../beam/dsls/sql/schema/BeamTableUtils.java  | 71 +++++++++-----
 .../dsls/sql/transform/BeamSQLProjectFn.java  |  3 +-
 .../beam/dsls/sql/planner/BasePlanner.java    |  2 +-
 .../BeamPlannerAggregationSubmitTest.java     | 13 ++-
 .../sql/planner/BeamPlannerSubmitTest.java    |  8 +-
 .../dsls/sql/planner/MockedBeamSQLTable.java  |  9 +-
 .../beam/dsls/sql/rel/BeamSortRelTest.java    | 22 +++--
 .../beam/dsls/sql/rel/BeamValuesRelTest.java  | 95 +++++++++++++++++++
 15 files changed, 318 insertions(+), 51 deletions(-)
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java
 create mode 100644 dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamValuesRule.java
 create mode 100644 dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamValuesRelTest.java

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
index be388aacbac1..9dcf003be423 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSQLFnExecutor.java
@@ -63,7 +63,9 @@
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.schema.impl.ScalarFunctionImpl;
 import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+import org.apache.calcite.util.NlsString;
 
 /**
  * Executor based on {@link BeamSqlExpression} and {@link BeamSqlPrimitive}.
@@ -99,8 +101,14 @@ public BeamSQLFnExecutor(BeamRelNode relNode) {
   static BeamSqlExpression buildExpression(RexNode rexNode) {
     if (rexNode instanceof RexLiteral) {
       RexLiteral node = (RexLiteral) rexNode;
-
-      return BeamSqlPrimitive.of(node.getTypeName(), node.getValue());
+      // NlsString is not serializable, we need to convert
+      // it to string explicitly.
+      if (SqlTypeName.CHAR_TYPES.contains(node.getTypeName())
+          && node.getValue() instanceof NlsString) {
+        return BeamSqlPrimitive.of(node.getTypeName(), ((NlsString) node.getValue()).getValue());
+      } else {
+        return BeamSqlPrimitive.of(node.getTypeName(), node.getValue());
+      }
     } else if (rexNode instanceof RexInputRef) {
       RexInputRef node = (RexInputRef) rexNode;
       return new BeamSqlInputRefExpression(node.getType().getSqlTypeName(), node.getIndex());
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
index a5938f3f71eb..bc18c5eb2eec 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
@@ -21,6 +21,7 @@
 import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.List;
+
 import org.apache.beam.dsls.sql.exception.BeamInvalidOperatorException;
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
index 2cac5aeda2b9..1ad62bcd6b73 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamRuleSets.java
@@ -28,6 +28,7 @@
 import org.apache.beam.dsls.sql.rule.BeamIOSourceRule;
 import org.apache.beam.dsls.sql.rule.BeamProjectRule;
 import org.apache.beam.dsls.sql.rule.BeamSortRule;
+import org.apache.beam.dsls.sql.rule.BeamValuesRule;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.tools.RuleSet;
@@ -41,7 +42,7 @@ public class BeamRuleSets {
   private static final ImmutableSet<RelOptRule> calciteToBeamConversionRules = ImmutableSet
       .<RelOptRule>builder().add(BeamIOSourceRule.INSTANCE, BeamProjectRule.INSTANCE,
           BeamFilterRule.INSTANCE, BeamIOSinkRule.INSTANCE,
-          BeamAggregationRule.INSTANCE, BeamSortRule.INSTANCE)
+          BeamAggregationRule.INSTANCE, BeamSortRule.INSTANCE, BeamValuesRule.INSTANCE)
       .build();
 
   public static RuleSet[] getRuleSets() {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java
new file mode 100644
index 000000000000..4fbe7ec4ee39
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.rel;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.planner.BeamPipelineCreator;
+import org.apache.beam.dsls.sql.planner.BeamSQLRelUtils;
+import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.schema.BeamTableUtils;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.core.Values;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexLiteral;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Values} node.
+ *
+ * <p>{@code BeamValuesRel} will be used in the following SQLs:
+ * <ul>
+ *   <li>{@code insert into t (name, desc) values ('hello', 'world')}</li>
+ *   <li>{@code select 1, '1', LOCALTIME}</li>
+ * </ul>
+ */
+public class BeamValuesRel extends Values implements BeamRelNode {
+
+  public BeamValuesRel(
+      RelOptCluster cluster,
+      RelDataType rowType,
+      ImmutableList<ImmutableList<RexLiteral>> tuples,
+      RelTraitSet traits) {
+    super(cluster, rowType, tuples, traits);
+
+  }
+
+  @Override public PCollection<BeamSQLRow> buildBeamPipeline(
+      BeamPipelineCreator planCreator) throws Exception {
+    List<BeamSQLRow> rows = new ArrayList<>(tuples.size());
+    String stageName = BeamSQLRelUtils.getStageName(this);
+    if (tuples.isEmpty()) {
+      throw new IllegalStateException("Values with empty tuples!");
+    }
+
+    BeamSQLRecordType beamSQLRecordType = BeamSQLRecordType.from(this.getRowType());
+    for (ImmutableList<RexLiteral> tuple : tuples) {
+      BeamSQLRow row = new BeamSQLRow(beamSQLRecordType);
+      for (int i = 0; i < tuple.size(); i++) {
+        BeamTableUtils.addFieldWithAutoTypeCasting(row, i, tuple.get(i).getValue());
+      }
+      rows.add(row);
+    }
+
+    return planCreator.getPipeline().apply(stageName, Create.of(rows));
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamValuesRule.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamValuesRule.java
new file mode 100644
index 000000000000..4ea9e609ba15
--- /dev/null
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rule/BeamValuesRule.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.rule;
+
+import org.apache.beam.dsls.sql.rel.BeamLogicalConvention;
+import org.apache.beam.dsls.sql.rel.BeamValuesRel;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Values;
+import org.apache.calcite.rel.logical.LogicalValues;
+
+/**
+ * {@code ConverterRule} to replace {@code Values} with {@code BeamValuesRel}.
+ */
+public class BeamValuesRule extends ConverterRule {
+  public static final BeamValuesRule INSTANCE = new BeamValuesRule();
+  private BeamValuesRule() {
+    super(LogicalValues.class, Convention.NONE,
+        BeamLogicalConvention.INSTANCE, "BeamValuesRule");
+  }
+
+  @Override public RelNode convert(RelNode rel) {
+    Values values = (Values) rel;
+    return new BeamValuesRel(
+        values.getCluster(),
+        values.getRowType(),
+        values.getTuples(),
+        values.getTraitSet().replace(BeamLogicalConvention.INSTANCE)
+    );
+  }
+}
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
index 7b6428ed2194..bc75eb17aee7 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
@@ -121,6 +121,7 @@ public void addField(int index, Object fieldValue) {
       }
       break;
     case VARCHAR:
+    case CHAR:
       if (!(fieldValue instanceof String)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
@@ -232,6 +233,7 @@ public Object getFieldValue(int fieldIdx) {
         return fieldValue;
       }
     case VARCHAR:
+    case CHAR:
       if (!(fieldValue instanceof String)) {
         throw new InvalidFieldException(
             String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index 0accb9a4f6a3..bfcb4874ac1c 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -85,6 +85,7 @@ public void encode(BeamSQLRow value, OutputStream outStream,
           longCoder.encode(value.getLong(idx), outStream, context.nested());
           break;
         case VARCHAR:
+        case CHAR:
           stringCoder.encode(value.getString(idx), outStream, context.nested());
           break;
         case TIMESTAMP:
@@ -134,6 +135,7 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
           record.addField(idx, longCoder.decode(inStream, context.nested()));
           break;
         case VARCHAR:
+        case CHAR:
           record.addField(idx, stringCoder.decode(inStream, context.nested()));
           break;
         case TIMESTAMP:
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
index bc622c2de9e9..c7397e14e643 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
@@ -21,9 +21,11 @@
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.StringWriter;
+import java.math.BigDecimal;
 
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.NlsString;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVPrinter;
@@ -72,33 +74,50 @@ public static String beamSQLRow2CsvLine(BeamSQLRow row, CSVFormat csvFormat) {
     return writer.toString();
   }
 
-  public static void addFieldWithAutoTypeCasting(BeamSQLRow row, int idx, String raw) {
+  public static void addFieldWithAutoTypeCasting(BeamSQLRow row, int idx, Object rawObj) {
+    if (rawObj == null) {
+      row.addField(idx, rawObj);
+      return;
+    }
+
     SqlTypeName columnType = row.getDataType().getFieldsType().get(idx);
-    switch (columnType) {
-      case TINYINT:
-        row.addField(idx, Byte.valueOf(raw));
-        break;
-      case SMALLINT:
-        row.addField(idx, Short.valueOf(raw));
-        break;
-      case INTEGER:
-        row.addField(idx, Integer.valueOf(raw));
-        break;
-      case BIGINT:
-        row.addField(idx, Long.valueOf(raw));
-        break;
-      case FLOAT:
-        row.addField(idx, Float.valueOf(raw));
-        break;
-      case DOUBLE:
-        row.addField(idx, Double.valueOf(raw));
-        break;
-      case VARCHAR:
-        row.addField(idx, raw);
-        break;
-      default:
-        throw new BeamSqlUnsupportedException(
-            String.format("Column type %s is not supported yet!", columnType));
+    // auto-casting for numberics
+    if ((rawObj instanceof String && SqlTypeName.NUMERIC_TYPES.contains(columnType))
+        || (rawObj instanceof BigDecimal && columnType != SqlTypeName.DECIMAL)) {
+      String raw = rawObj.toString();
+      switch (columnType) {
+        case TINYINT:
+          row.addField(idx, Byte.valueOf(raw));
+          break;
+        case SMALLINT:
+          row.addField(idx, Short.valueOf(raw));
+          break;
+        case INTEGER:
+          row.addField(idx, Integer.valueOf(raw));
+          break;
+        case BIGINT:
+          row.addField(idx, Long.valueOf(raw));
+          break;
+        case FLOAT:
+          row.addField(idx, Float.valueOf(raw));
+          break;
+        case DOUBLE:
+          row.addField(idx, Double.valueOf(raw));
+          break;
+        default:
+          throw new BeamSqlUnsupportedException(
+              String.format("Column type %s is not supported yet!", columnType));
+      }
+    } else if (SqlTypeName.CHAR_TYPES.contains(columnType)) {
+      // convert NlsString to String
+      if (rawObj instanceof NlsString) {
+        row.addField(idx, ((NlsString) rawObj).getValue());
+      } else {
+        row.addField(idx, rawObj);
+      }
+    } else {
+      // keep the origin
+      row.addField(idx, rawObj);
     }
   }
 }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
index 79dd67f74e92..ef4dc0fe063d 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/transform/BeamSQLProjectFn.java
@@ -22,6 +22,7 @@
 import org.apache.beam.dsls.sql.rel.BeamProjectRel;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
 import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.dsls.sql.schema.BeamTableUtils;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 
@@ -57,7 +58,7 @@ public void processElement(ProcessContext c, BoundedWindow window) {
     outRow.updateWindowRange(inputRecord, window);
 
     for (int idx = 0; idx < results.size(); ++idx) {
-      outRow.addField(idx, results.get(idx));
+      BeamTableUtils.addFieldWithAutoTypeCasting(outRow, idx, results.get(idx));
     }
 
     c.output(outRow);
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
index fe8a2364bb84..0d9d147c919f 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BasePlanner.java
@@ -41,7 +41,7 @@ public class BasePlanner {
   public static BeamSQLEnvironment runner = BeamSQLEnvironment.create();
 
   @BeforeClass
-  public static void prepare() {
+  public static void prepareClass() {
     runner.addTableMetadata("ORDER_DETAILS", getTable());
     runner.addTableMetadata("SUB_ORDER", getTable("127.0.0.1:9092", "sub_orders"));
     runner.addTableMetadata("SUB_ORDER_RAM", getTable());
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
index 22f1848deb46..ffc3e019f3aa 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerAggregationSubmitTest.java
@@ -21,6 +21,7 @@
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
+
 import org.apache.beam.dsls.sql.BeamSQLEnvironment;
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamSQLRecordType;
@@ -31,6 +32,7 @@
 import org.apache.calcite.rel.type.RelProtoDataType;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
@@ -47,11 +49,16 @@ public class BeamPlannerAggregationSubmitTest {
   public final TestPipeline pipeline = TestPipeline.create();
 
   @BeforeClass
-  public static void prepare() throws ParseException {
+  public static void prepareClass() throws ParseException {
     runner.addTableMetadata("ORDER_DETAILS", getOrderTable());
     runner.addTableMetadata("ORDER_SUMMARY", getSummaryTable());
   }
 
+  @Before
+  public void prepare() throws ParseException {
+    MockedBeamSQLTable.CONTENT.clear();
+  }
+
   private static BaseBeamTable getOrderTable() throws ParseException {
     final RelProtoDataType protoRowType = new RelProtoDataType() {
       @Override
@@ -118,7 +125,7 @@ public void selectWithWindowAggregation() throws Exception{
     pipeline.run().waitUntilFinish();
 
     Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
-    BeamSQLRow result = MockedBeamSQLTable.CONTENT.get(0);
+    BeamSQLRow result = MockedBeamSQLTable.CONTENT.peek();
     Assert.assertEquals(1, result.getInteger(0));
     Assert.assertEquals(format.parse("2017-01-01 01:00:00"), result.getDate(1));
     Assert.assertEquals(1L, result.getLong(2));
@@ -136,6 +143,6 @@ public void selectWithoutWindowAggregation() throws Exception{
 
     Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
     Assert.assertEquals("site_id=0,agg_hour=null,size=3",
-        MockedBeamSQLTable.CONTENT.get(0).valueInString());
+        MockedBeamSQLTable.CONTENT.peek().valueInString());
   }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
index 17cea274a9dc..7219d114f61b 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/BeamPlannerSubmitTest.java
@@ -21,6 +21,7 @@
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.values.PCollection;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
@@ -32,6 +33,11 @@ public class BeamPlannerSubmitTest extends BasePlanner {
   @Rule
   public final TestPipeline pipeline = TestPipeline.create();
 
+  @Before
+  public void prepare() {
+    MockedBeamSQLTable.CONTENT.clear();
+  }
+
   @Test
   public void insertSelectFilter() throws Exception {
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price) SELECT "
@@ -43,7 +49,7 @@ public void insertSelectFilter() throws Exception {
     pipeline.run().waitUntilFinish();
 
     Assert.assertTrue(MockedBeamSQLTable.CONTENT.size() == 1);
-    Assert.assertTrue(MockedBeamSQLTable.CONTENT.get(0).valueInString()
+    Assert.assertTrue(MockedBeamSQLTable.CONTENT.peek().valueInString()
         .contains("order_id=12345,site_id=0,price=20.5,order_time="));
   }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
index 78fd05534a52..561f4beb6e35 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/planner/MockedBeamSQLTable.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.apache.beam.dsls.sql.schema.BaseBeamTable;
 import org.apache.beam.dsls.sql.schema.BeamIOType;
@@ -43,7 +44,7 @@
  */
 public class MockedBeamSQLTable extends BaseBeamTable {
 
-  public static final List<BeamSQLRow> CONTENT = new ArrayList<>();
+  public static final ConcurrentLinkedQueue<BeamSQLRow> CONTENT = new ConcurrentLinkedQueue<>();
 
   private List<BeamSQLRow> inputRecords;
 
@@ -142,12 +143,6 @@ public static class OutputStore extends PTransform<PCollection<BeamSQLRow>, PDon
     @Override
     public PDone expand(PCollection<BeamSQLRow> input) {
       input.apply(ParDo.of(new DoFn<BeamSQLRow, Void>() {
-
-        @Setup
-        public void setup() {
-          CONTENT.clear();
-        }
-
         @ProcessElement
         public void processElement(ProcessContext c) {
           CONTENT.add(c.element());
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
index 864d4b7c97fe..4935c3b2df3c 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamSortRelTest.java
@@ -18,8 +18,10 @@
 
 package org.apache.beam.dsls.sql.rel;
 
+import java.util.Collection;
 import java.util.Date;
-import java.util.List;
+import java.util.Iterator;
+
 import org.apache.beam.dsls.sql.BeamSQLEnvironment;
 import org.apache.beam.dsls.sql.exception.BeamSqlUnsupportedException;
 import org.apache.beam.dsls.sql.planner.MockedBeamSQLTable;
@@ -27,6 +29,7 @@
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
@@ -62,7 +65,6 @@ public class BeamSortRelTest {
 
   @Test
   public void testOrderBy_basic() throws Exception {
-    prepare();
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
         + " order_id, site_id, price "
         + "FROM ORDER_DETAILS "
@@ -162,7 +164,6 @@ public void testOrderBy_nullsLast() throws Exception {
 
   @Test
   public void testOrderBy_with_offset() throws Exception {
-    prepare();
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
         + " order_id, site_id, price "
         + "FROM ORDER_DETAILS "
@@ -186,7 +187,6 @@ public void testOrderBy_with_offset() throws Exception {
 
   @Test
   public void testOrderBy_bigFetch() throws Exception {
-    prepare();
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id, price)  SELECT "
         + " order_id, site_id, price "
         + "FROM ORDER_DETAILS "
@@ -216,7 +216,6 @@ public void testOrderBy_bigFetch() throws Exception {
 
   @Test(expected = BeamSqlUnsupportedException.class)
   public void testOrderBy_exception() throws Exception {
-    prepare();
     String sql = "INSERT INTO SUB_ORDER_RAM(order_id, site_id)  SELECT "
         + " order_id, COUNT(*) "
         + "FROM ORDER_DETAILS "
@@ -227,15 +226,20 @@ public void testOrderBy_exception() throws Exception {
     runner.compileBeamPipeline(sql, pipeline);
   }
 
-  public static void prepare() {
+  @Before
+  public void prepare() {
     runner.addTableMetadata("ORDER_DETAILS", orderDetailTable);
     runner.addTableMetadata("SUB_ORDER_RAM", subOrderRamTable);
+    MockedBeamSQLTable.CONTENT.clear();
   }
 
-  private void assertEquals(List<BeamSQLRow> rows1, List<BeamSQLRow> rows2) {
+  private void assertEquals(Collection<BeamSQLRow> rows1, Collection<BeamSQLRow> rows2) {
     Assert.assertEquals(rows1.size(), rows2.size());
-    for (int i = 0; i < rows1.size(); i++) {
-      Assert.assertEquals(rows1.get(i), rows2.get(i));
+
+    Iterator<BeamSQLRow> it1 = rows1.iterator();
+    Iterator<BeamSQLRow> it2 = rows2.iterator();
+    while (it1.hasNext()) {
+      Assert.assertEquals(it1.next(), it2.next());
     }
   }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamValuesRelTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamValuesRelTest.java
new file mode 100644
index 000000000000..d4e1db2bfe17
--- /dev/null
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamValuesRelTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.rel;
+
+import org.apache.beam.dsls.sql.BeamSQLEnvironment;
+import org.apache.beam.dsls.sql.planner.MockedBeamSQLTable;
+import org.apache.beam.dsls.sql.schema.BeamSQLRow;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamValuesRel}.
+ */
+public class BeamValuesRelTest {
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
+  public static BeamSQLEnvironment runner = BeamSQLEnvironment.create();
+  private static MockedBeamSQLTable stringTable = MockedBeamSQLTable
+      .of(SqlTypeName.VARCHAR, "name",
+          SqlTypeName.VARCHAR, "description");
+
+  private static MockedBeamSQLTable intTable = MockedBeamSQLTable
+      .of(SqlTypeName.INTEGER, "c0",
+          SqlTypeName.INTEGER, "c1");
+
+  @Test
+  public void testValues() throws Exception {
+    String sql = "insert into string_table(name, description) values "
+        + "('hello', 'world'), ('james', 'bond')";
+    PCollection<BeamSQLRow> rows = runner.compileBeamPipeline(sql, pipeline);
+    PAssert.that(rows).containsInAnyOrder(MockedBeamSQLTable.of(
+        SqlTypeName.VARCHAR, "name",
+        SqlTypeName.VARCHAR, "description",
+        "hello", "world",
+        "james", "bond").getInputRecords());
+    pipeline.run();
+  }
+
+  @Test
+  public void testValues_castInt() throws Exception {
+    String sql = "insert into int_table (c0, c1) values(cast(1 as int), cast(2 as int))";
+    PCollection<BeamSQLRow> rows = runner.compileBeamPipeline(sql, pipeline);
+    PAssert.that(rows).containsInAnyOrder(MockedBeamSQLTable.of(
+        SqlTypeName.INTEGER, "c0",
+        SqlTypeName.INTEGER, "c1",
+        1, 2
+    ).getInputRecords());
+    pipeline.run();
+  }
+
+  @Test
+  public void testValues_onlySelect() throws Exception {
+    String sql = "select 1, '1'";
+    PCollection<BeamSQLRow> rows = runner.compileBeamPipeline(sql, pipeline);
+    PAssert.that(rows).containsInAnyOrder(MockedBeamSQLTable.of(
+        SqlTypeName.INTEGER, "EXPR$0",
+        SqlTypeName.CHAR, "EXPR$1",
+        1, "1"
+    ).getInputRecords());
+    pipeline.run();
+  }
+
+  @BeforeClass
+  public static void prepareClass() {
+    runner.addTableMetadata("string_table", stringTable);
+    runner.addTableMetadata("int_table", intTable);
+  }
+
+  @Before
+  public void prepare() {
+    MockedBeamSQLTable.CONTENT.clear();
+  }
+}

From 67a24497f937c2f7bfb9fe18f90d868ee62019ed Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Wed, 17 May 2017 22:48:00 +0800
Subject: [PATCH 386/387] [BEAM-2310] Support encoding/decoding of TIME and new
 DECIMAL data type

---
 .../beam/dsls/sql/schema/BeamSQLRow.java      | 268 ++++++++++--------
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java |  23 +-
 .../dsls/sql/schema/BeamSqlRowCoderTest.java  |  37 ++-
 3 files changed, 193 insertions(+), 135 deletions(-)

diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
index bc75eb17aee7..ca045c840f0d 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRow.java
@@ -18,6 +18,7 @@
 package org.apache.beam.dsls.sql.schema;
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.GregorianCalendar;
@@ -84,63 +85,69 @@ public void addField(int index, Object fieldValue) {
 
     SqlTypeName fieldType = dataType.getFieldsType().get(index);
     switch (fieldType) {
-    case INTEGER:
-      if (!(fieldValue instanceof Integer)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case SMALLINT:
-      if (!(fieldValue instanceof Short)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case TINYINT:
-      if (!(fieldValue instanceof Byte)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case DOUBLE:
-      if (!(fieldValue instanceof Double)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case BIGINT:
-      if (!(fieldValue instanceof Long)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case FLOAT:
-      if (!(fieldValue instanceof Float)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case VARCHAR:
-    case CHAR:
-      if (!(fieldValue instanceof String)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case TIME:
-      if (!(fieldValue instanceof GregorianCalendar)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    case TIMESTAMP:
-      if (!(fieldValue instanceof Date)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      }
-      break;
-    default:
-      throw new UnsupportedDataTypeException(fieldType);
+      case INTEGER:
+        if (!(fieldValue instanceof Integer)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case SMALLINT:
+        if (!(fieldValue instanceof Short)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case TINYINT:
+        if (!(fieldValue instanceof Byte)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case DOUBLE:
+        if (!(fieldValue instanceof Double)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case BIGINT:
+        if (!(fieldValue instanceof Long)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case FLOAT:
+        if (!(fieldValue instanceof Float)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case DECIMAL:
+        if (!(fieldValue instanceof BigDecimal)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case VARCHAR:
+      case CHAR:
+        if (!(fieldValue instanceof String)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case TIME:
+        if (!(fieldValue instanceof GregorianCalendar)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      case TIMESTAMP:
+        if (!(fieldValue instanceof Date)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        }
+        break;
+      default:
+        throw new UnsupportedDataTypeException(fieldType);
     }
     dataValues.set(index, fieldValue);
   }
@@ -177,6 +184,14 @@ public Date getDate(int idx) {
     return (Date) getFieldValue(idx);
   }
 
+  public GregorianCalendar getGregorianCalendar(int idx) {
+    return (GregorianCalendar) getFieldValue(idx);
+  }
+
+  public BigDecimal getBigDecimal(int idx) {
+    return (BigDecimal) getFieldValue(idx);
+  }
+
   public Object getFieldValue(String fieldName) {
     return getFieldValue(dataType.getFieldsName().indexOf(fieldName));
   }
@@ -190,72 +205,79 @@ public Object getFieldValue(int fieldIdx) {
     SqlTypeName fieldType = dataType.getFieldsType().get(fieldIdx);
 
     switch (fieldType) {
-    case INTEGER:
-      if (!(fieldValue instanceof Integer)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case SMALLINT:
-      if (!(fieldValue instanceof Short)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case TINYINT:
-      if (!(fieldValue instanceof Byte)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case DOUBLE:
-      if (!(fieldValue instanceof Double)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case BIGINT:
-      if (!(fieldValue instanceof Long)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case FLOAT:
-      if (!(fieldValue instanceof Float)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case VARCHAR:
-    case CHAR:
-      if (!(fieldValue instanceof String)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case TIME:
-      if (!(fieldValue instanceof GregorianCalendar)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    case TIMESTAMP:
-      if (!(fieldValue instanceof Date)) {
-        throw new InvalidFieldException(
-            String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
-      } else {
-        return fieldValue;
-      }
-    default:
-      throw new UnsupportedDataTypeException(fieldType);
+      case INTEGER:
+        if (!(fieldValue instanceof Integer)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case SMALLINT:
+        if (!(fieldValue instanceof Short)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case TINYINT:
+        if (!(fieldValue instanceof Byte)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case DOUBLE:
+        if (!(fieldValue instanceof Double)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case DECIMAL:
+        if (!(fieldValue instanceof BigDecimal)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case BIGINT:
+        if (!(fieldValue instanceof Long)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case FLOAT:
+        if (!(fieldValue instanceof Float)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case VARCHAR:
+      case CHAR:
+        if (!(fieldValue instanceof String)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case TIME:
+        if (!(fieldValue instanceof GregorianCalendar)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      case TIMESTAMP:
+        if (!(fieldValue instanceof Date)) {
+          throw new InvalidFieldException(
+              String.format("[%s] doesn't match type [%s]", fieldValue, fieldType));
+        } else {
+          return fieldValue;
+        }
+      default:
+        throw new UnsupportedDataTypeException(fieldType);
     }
   }
 
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index bfcb4874ac1c..0bfe467328c3 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -21,7 +21,10 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Date;
+import java.util.GregorianCalendar;
 import java.util.List;
+
+import org.apache.beam.sdk.coders.BigDecimalCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.Coder;
@@ -46,6 +49,7 @@ public class BeamSqlRowCoder extends StandardCoder<BeamSQLRow>{
   private static final BigEndianLongCoder longCoder = BigEndianLongCoder.of();
   private static final DoubleCoder doubleCoder = DoubleCoder.of();
   private static final InstantCoder instantCoder = InstantCoder.of();
+  private static final BigDecimalCoder bigDecimalCoder = BigDecimalCoder.of();
 
   private static final BeamSqlRowCoder INSTANCE = new BeamSqlRowCoder();
   private BeamSqlRowCoder(){}
@@ -81,6 +85,9 @@ public void encode(BeamSQLRow value, OutputStream outStream,
         case FLOAT:
           doubleCoder.encode((double) value.getFloat(idx), outStream, context.nested());
           break;
+        case DECIMAL:
+          bigDecimalCoder.encode(value.getBigDecimal(idx), outStream, context.nested());
+          break;
         case BIGINT:
           longCoder.encode(value.getLong(idx), outStream, context.nested());
           break;
@@ -88,8 +95,12 @@ public void encode(BeamSQLRow value, OutputStream outStream,
         case CHAR:
           stringCoder.encode(value.getString(idx), outStream, context.nested());
           break;
+        case TIME:
+          longCoder.encode(value.getGregorianCalendar(idx).getTime().getTime(),
+              outStream, context.nested());
+          break;
         case TIMESTAMP:
-          longCoder.encode(value.getDate(idx).getTime(), outStream, context);
+          longCoder.encode(value.getDate(idx).getTime(), outStream, context.nested());
           break;
 
         default:
@@ -134,12 +145,20 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
         case BIGINT:
           record.addField(idx, longCoder.decode(inStream, context.nested()));
           break;
+        case DECIMAL:
+          record.addField(idx, bigDecimalCoder.decode(inStream, context.nested()));
+          break;
         case VARCHAR:
         case CHAR:
           record.addField(idx, stringCoder.decode(inStream, context.nested()));
           break;
+        case TIME:
+          GregorianCalendar calendar = new GregorianCalendar();
+          calendar.setTime(new Date(longCoder.decode(inStream, context.nested())));
+          record.addField(idx, calendar);
+          break;
         case TIMESTAMP:
-          record.addField(idx, new Date(longCoder.decode(inStream, context)));
+          record.addField(idx, new Date(longCoder.decode(inStream, context.nested())));
           break;
 
         default:
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java
index f2077947ec3a..bc6343b92b70 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoderTest.java
@@ -18,6 +18,10 @@
 
 package org.apache.beam.dsls.sql.schema;
 
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.GregorianCalendar;
+
 import org.apache.beam.sdk.testing.CoderProperties;
 import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
 import org.apache.calcite.rel.type.RelDataType;
@@ -38,11 +42,16 @@ public void encodeAndDecode() throws Exception {
       @Override
       public RelDataType apply(RelDataTypeFactory a0) {
         return a0.builder()
-            .add("id", SqlTypeName.INTEGER)
-            .add("order_id", SqlTypeName.BIGINT)
-            .add("price", SqlTypeName.FLOAT)
-            .add("amount", SqlTypeName.DOUBLE)
-            .add("user_name", SqlTypeName.VARCHAR)
+            .add("col_tinyint", SqlTypeName.TINYINT)
+            .add("col_smallint", SqlTypeName.SMALLINT)
+            .add("col_integer", SqlTypeName.INTEGER)
+            .add("col_bigint", SqlTypeName.BIGINT)
+            .add("col_float", SqlTypeName.FLOAT)
+            .add("col_double", SqlTypeName.DOUBLE)
+            .add("col_decimal", SqlTypeName.DECIMAL)
+            .add("col_string_varchar", SqlTypeName.VARCHAR)
+            .add("col_time", SqlTypeName.TIME)
+            .add("col_timestamp", SqlTypeName.TIMESTAMP)
             .build();
       }
     };
@@ -51,11 +60,19 @@ public RelDataType apply(RelDataTypeFactory a0) {
         protoRowType.apply(new JavaTypeFactoryImpl(
         RelDataTypeSystem.DEFAULT)));
     BeamSQLRow row = new BeamSQLRow(beamSQLRecordType);
-    row.addField(0, 1);
-    row.addField(1, 1L);
-    row.addField(2, 1.1F);
-    row.addField(3, 1.1);
-    row.addField(4, "hello");
+    row.addField("col_tinyint", Byte.valueOf("1"));
+    row.addField("col_smallint", Short.valueOf("1"));
+    row.addField("col_integer", 1);
+    row.addField("col_bigint", 1L);
+    row.addField("col_float", 1.1F);
+    row.addField("col_double", 1.1);
+    row.addField("col_decimal", BigDecimal.ZERO);
+    row.addField("col_string_varchar", "hello");
+    GregorianCalendar calendar = new GregorianCalendar();
+    calendar.setTime(new Date());
+    row.addField("col_time", calendar);
+    row.addField("col_timestamp", new Date());
+
 
     BeamSqlRowCoder coder = BeamSqlRowCoder.of();
     CoderProperties.coderDecodeEncodeEqual(coder, row);

From a97ac2767c5eb04663535dd428fa0f4b679d3d86 Mon Sep 17 00:00:00 2001
From: James Xu <xumingmingv@gmail.com>
Date: Sun, 28 May 2017 00:59:54 +0800
Subject: [PATCH 387/387] bump versions to 2.1.0-SNAPSHOT with necessary code
 changes

---
 dsls/pom.xml                                  |  4 +-
 dsls/sql/pom.xml                              | 19 +++--
 .../dsls/sql/planner/BeamPipelineCreator.java |  4 +-
 .../sql/schema/BeamSQLRecordTypeCoder.java    | 35 ++++------
 .../beam/dsls/sql/schema/BeamSqlRowCoder.java | 70 +++++++++----------
 .../dsls/sql/schema/kafka/BeamKafkaTable.java | 18 +++--
 .../sql/schema/text/BeamTextCSVTable.java     |  2 +-
 .../schema/text/BeamTextCSVTableIOWriter.java |  2 +-
 .../BeamAggregationTransformTest.java         |  4 +-
 9 files changed, 80 insertions(+), 78 deletions(-)

diff --git a/dsls/pom.xml b/dsls/pom.xml
index 6f9d6353596b..23d122943579 100644
--- a/dsls/pom.xml
+++ b/dsls/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -52,7 +52,7 @@
         </plugin>
       </plugins>
     </pluginManagement>
-    
+
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
diff --git a/dsls/sql/pom.xml b/dsls/sql/pom.xml
index bc658e6969ee..51ceaf7422ab 100644
--- a/dsls/sql/pom.xml
+++ b/dsls/sql/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.beam</groupId>
     <artifactId>beam-dsls-parent</artifactId>
-    <version>0.7.0-SNAPSHOT</version>
+    <version>2.1.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>beam-dsls-sql</artifactId>
@@ -122,32 +122,32 @@
       <dependency>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-sdks-java-core</artifactId>
-        <version>0.6.0</version>
+        <version>2.1.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-runners-direct-java</artifactId>
-        <version>0.6.0</version>
+        <version>2.1.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-sdks-java-io-kafka</artifactId>
-        <version>0.6.0</version>
+        <version>2.1.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-runners-core-java</artifactId>
-        <version>0.6.0</version>
+        <version>2.1.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-sdks-common-runner-api</artifactId>
-        <version>0.6.0</version>
+        <version>2.1.0-SNAPSHOT</version>
       </dependency>
       <dependency>
         <groupId>org.apache.beam</groupId>
         <artifactId>beam-runners-core-construction-java</artifactId>
-        <version>0.6.0</version>
+        <version>2.1.0-SNAPSHOT</version>
       </dependency>
     </dependencies>
   </dependencyManagement>
@@ -182,6 +182,11 @@
       <artifactId>beam-sdks-java-io-kafka</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-clients</artifactId>
+      <version>0.9.0.1</version>
+    </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
index 1f3ba58de2df..bf273412a861 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java
@@ -48,8 +48,8 @@ public BeamPipelineCreator(Map<String, BaseBeamTable> sourceTables, Pipeline pip
     this.pipeline = pipeline;
 
     CoderRegistry cr = pipeline.getCoderRegistry();
-    cr.registerCoder(BeamSQLRow.class, BeamSqlRowCoder.of());
-    cr.registerCoder(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
+    cr.registerCoderForClass(BeamSQLRow.class, BeamSqlRowCoder.of());
+    cr.registerCoderForClass(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
   }
 
   public Map<String, BaseBeamTable> getSourceTables() {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
index b88a19557280..f86fa99cb9ff 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSQLRecordTypeCoder.java
@@ -20,10 +20,10 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.List;
+
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.StandardCoder;
+import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.calcite.sql.type.SqlTypeName;
@@ -32,7 +32,7 @@
  * A {@link Coder} for {@link BeamSQLRecordType}.
  *
  */
-public class BeamSQLRecordTypeCoder extends StandardCoder<BeamSQLRecordType> {
+public class BeamSQLRecordTypeCoder extends CustomCoder<BeamSQLRecordType> {
   private static final StringUtf8Coder stringCoder = StringUtf8Coder.of();
   private static final VarIntCoder intCoder = VarIntCoder.of();
 
@@ -44,41 +44,34 @@ public static BeamSQLRecordTypeCoder of() {
   }
 
   @Override
-  public void encode(BeamSQLRecordType value, OutputStream outStream,
-      org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
-    Context nested = context.nested();
-    intCoder.encode(value.size(), outStream, nested);
+  public void encode(BeamSQLRecordType value, OutputStream outStream)
+      throws CoderException, IOException {
+    intCoder.encode(value.size(), outStream);
     for (String fieldName : value.getFieldsName()) {
-      stringCoder.encode(fieldName, outStream, nested);
+      stringCoder.encode(fieldName, outStream);
     }
     for (SqlTypeName fieldType : value.getFieldsType()) {
-      stringCoder.encode(fieldType.name(), outStream, nested);
+      stringCoder.encode(fieldType.name(), outStream);
     }
     //add a dummy field to indicate the end of record
-    intCoder.encode(value.size(), outStream, context);
+    intCoder.encode(value.size(), outStream);
   }
 
   @Override
-  public BeamSQLRecordType decode(InputStream inStream,
-      org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
+  public BeamSQLRecordType decode(InputStream inStream) throws CoderException, IOException {
     BeamSQLRecordType typeRecord = new BeamSQLRecordType();
-    int size = intCoder.decode(inStream, context.nested());
+    int size = intCoder.decode(inStream);
     for (int idx = 0; idx < size; ++idx) {
-      typeRecord.getFieldsName().add(stringCoder.decode(inStream, context.nested()));
+      typeRecord.getFieldsName().add(stringCoder.decode(inStream));
     }
     for (int idx = 0; idx < size; ++idx) {
       typeRecord.getFieldsType().add(
-          SqlTypeName.valueOf(stringCoder.decode(inStream, context.nested())));
+          SqlTypeName.valueOf(stringCoder.decode(inStream)));
     }
-    intCoder.decode(inStream, context);
+    intCoder.decode(inStream);
     return typeRecord;
   }
 
-  @Override
-  public List<? extends Coder<?>> getCoderArguments() {
-    return null;
-  }
-
   @Override
   public void verifyDeterministic()
       throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
index 0bfe467328c3..c5c044e330ca 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlRowCoder.java
@@ -29,17 +29,17 @@
 import org.apache.beam.sdk.coders.BigEndianLongCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
 import org.apache.beam.sdk.coders.DoubleCoder;
 import org.apache.beam.sdk.coders.InstantCoder;
 import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.coders.StandardCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 
 /**
  * A {@link Coder} encodes {@link BeamSQLRow}.
  *
  */
-public class BeamSqlRowCoder extends StandardCoder<BeamSQLRow>{
+public class BeamSqlRowCoder extends CustomCoder<BeamSQLRow> {
   private static final BeamSQLRecordTypeCoder recordTypeCoder = BeamSQLRecordTypeCoder.of();
 
   private static final ListCoder<Integer> listCoder = ListCoder.of(BigEndianIntegerCoder.of());
@@ -59,10 +59,9 @@ public static BeamSqlRowCoder of() {
   }
 
   @Override
-  public void encode(BeamSQLRow value, OutputStream outStream,
-      org.apache.beam.sdk.coders.Coder.Context context) throws CoderException, IOException {
-    recordTypeCoder.encode(value.getDataType(), outStream, context.nested());
-    listCoder.encode(value.getNullFields(), outStream, context.nested());
+  public void encode(BeamSQLRow value, OutputStream outStream) throws CoderException, IOException {
+    recordTypeCoder.encode(value.getDataType(), outStream);
+    listCoder.encode(value.getNullFields(), outStream);
 
     for (int idx = 0; idx < value.size(); ++idx) {
       if (value.getNullFields().contains(idx)) {
@@ -71,36 +70,36 @@ public void encode(BeamSQLRow value, OutputStream outStream,
 
       switch (value.getDataType().getFieldsType().get(idx)) {
         case INTEGER:
-          intCoder.encode(value.getInteger(idx), outStream, context.nested());
+          intCoder.encode(value.getInteger(idx), outStream);
           break;
         case SMALLINT:
-          intCoder.encode((int) value.getShort(idx), outStream, context.nested());
+          intCoder.encode((int) value.getShort(idx), outStream);
           break;
         case TINYINT:
-          intCoder.encode((int) value.getByte(idx), outStream, context.nested());
+          intCoder.encode((int) value.getByte(idx), outStream);
           break;
         case DOUBLE:
-          doubleCoder.encode(value.getDouble(idx), outStream, context.nested());
+          doubleCoder.encode(value.getDouble(idx), outStream);
           break;
         case FLOAT:
-          doubleCoder.encode((double) value.getFloat(idx), outStream, context.nested());
+          doubleCoder.encode((double) value.getFloat(idx), outStream);
           break;
         case DECIMAL:
-          bigDecimalCoder.encode(value.getBigDecimal(idx), outStream, context.nested());
+          bigDecimalCoder.encode(value.getBigDecimal(idx), outStream);
           break;
         case BIGINT:
-          longCoder.encode(value.getLong(idx), outStream, context.nested());
+          longCoder.encode(value.getLong(idx), outStream);
           break;
         case VARCHAR:
         case CHAR:
-          stringCoder.encode(value.getString(idx), outStream, context.nested());
+          stringCoder.encode(value.getString(idx), outStream);
           break;
         case TIME:
           longCoder.encode(value.getGregorianCalendar(idx).getTime().getTime(),
-              outStream, context.nested());
+              outStream);
           break;
         case TIMESTAMP:
-          longCoder.encode(value.getDate(idx).getTime(), outStream, context.nested());
+          longCoder.encode(value.getDate(idx).getTime(), outStream);
           break;
 
         default:
@@ -108,15 +107,15 @@ public void encode(BeamSQLRow value, OutputStream outStream,
       }
     }
 
-    instantCoder.encode(value.getWindowStart(), outStream, context.nested());
-    instantCoder.encode(value.getWindowEnd(), outStream, context);
+    instantCoder.encode(value.getWindowStart(), outStream);
+    instantCoder.encode(value.getWindowEnd(), outStream);
   }
 
   @Override
-  public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.Context context)
+  public BeamSQLRow decode(InputStream inStream)
       throws CoderException, IOException {
-    BeamSQLRecordType type = recordTypeCoder.decode(inStream, context.nested());
-    List<Integer> nullFields = listCoder.decode(inStream, context.nested());
+    BeamSQLRecordType type = recordTypeCoder.decode(inStream);
+    List<Integer> nullFields = listCoder.decode(inStream);
 
     BeamSQLRow record = new BeamSQLRow(type);
     record.setNullFields(nullFields);
@@ -128,37 +127,37 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
 
       switch (type.getFieldsType().get(idx)) {
         case INTEGER:
-          record.addField(idx, intCoder.decode(inStream, context.nested()));
+          record.addField(idx, intCoder.decode(inStream));
           break;
         case SMALLINT:
-          record.addField(idx, intCoder.decode(inStream, context.nested()).shortValue());
+          record.addField(idx, intCoder.decode(inStream).shortValue());
           break;
         case TINYINT:
-          record.addField(idx, intCoder.decode(inStream, context.nested()).byteValue());
+          record.addField(idx, intCoder.decode(inStream).byteValue());
           break;
         case DOUBLE:
-          record.addField(idx, doubleCoder.decode(inStream, context.nested()));
+          record.addField(idx, doubleCoder.decode(inStream));
           break;
         case FLOAT:
-          record.addField(idx, doubleCoder.decode(inStream, context.nested()).floatValue());
+          record.addField(idx, doubleCoder.decode(inStream).floatValue());
           break;
         case BIGINT:
-          record.addField(idx, longCoder.decode(inStream, context.nested()));
+          record.addField(idx, longCoder.decode(inStream));
           break;
         case DECIMAL:
-          record.addField(idx, bigDecimalCoder.decode(inStream, context.nested()));
+          record.addField(idx, bigDecimalCoder.decode(inStream));
           break;
         case VARCHAR:
         case CHAR:
-          record.addField(idx, stringCoder.decode(inStream, context.nested()));
+          record.addField(idx, stringCoder.decode(inStream));
           break;
         case TIME:
           GregorianCalendar calendar = new GregorianCalendar();
-          calendar.setTime(new Date(longCoder.decode(inStream, context.nested())));
+          calendar.setTime(new Date(longCoder.decode(inStream)));
           record.addField(idx, calendar);
           break;
         case TIMESTAMP:
-          record.addField(idx, new Date(longCoder.decode(inStream, context.nested())));
+          record.addField(idx, new Date(longCoder.decode(inStream)));
           break;
 
         default:
@@ -166,17 +165,12 @@ public BeamSQLRow decode(InputStream inStream, org.apache.beam.sdk.coders.Coder.
       }
     }
 
-    record.setWindowStart(instantCoder.decode(inStream, context.nested()));
-    record.setWindowEnd(instantCoder.decode(inStream, context));
+    record.setWindowStart(instantCoder.decode(inStream));
+    record.setWindowEnd(instantCoder.decode(inStream));
 
     return record;
   }
 
-  @Override
-  public List<? extends Coder<?>> getCoderArguments() {
-    return null;
-  }
-
   @Override
   public void verifyDeterministic()
       throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
index 7342cee38714..2be2ff4e2d08 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/kafka/BeamKafkaTable.java
@@ -33,6 +33,8 @@
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
 
 /**
  * {@code BeamKafkaTable} represent a Kafka topic, as source or target. Need to
@@ -76,8 +78,15 @@ public BeamIOType getSourceType() {
   public PCollection<BeamSQLRow> buildIOReader(Pipeline pipeline) {
     return PBegin.in(pipeline).apply("read",
             KafkaIO.<byte[], byte[]>read().withBootstrapServers(bootstrapServers).withTopics(topics)
-                .updateConsumerProperties(configUpdates).withKeyCoder(ByteArrayCoder.of())
-                .withValueCoder(ByteArrayCoder.of()).withoutMetadata())
+                .updateConsumerProperties(configUpdates)
+                .withKeyDeserializerAndCoder(
+                    ByteArrayDeserializer.class,
+                    ByteArrayCoder.of()
+                )
+                .withValueDeserializerAndCoder(
+                    ByteArrayDeserializer.class,
+                    ByteArrayCoder.of()
+                ).withoutMetadata())
             .apply("in_format", getPTransformForInput());
   }
 
@@ -91,8 +100,9 @@ public PTransform<? super PCollection<BeamSQLRow>, PDone> buildIOWriter() {
       public PDone expand(PCollection<BeamSQLRow> input) {
         return input.apply("out_reformat", getPTransformForOutput()).apply("persistent",
             KafkaIO.<byte[], byte[]>write().withBootstrapServers(bootstrapServers)
-                .withTopic(topics.get(0)).withKeyCoder(ByteArrayCoder.of())
-                .withValueCoder(ByteArrayCoder.of()));
+                .withTopic(topics.get(0))
+                .withKeySerializer(ByteArraySerializer.class)
+                .withValueSerializer(ByteArraySerializer.class));
       }
     };
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
index 6b2128945b16..41742c7ba30f 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTable.java
@@ -58,7 +58,7 @@ public BeamTextCSVTable(RelProtoDataType protoDataType, String filePattern,
 
   @Override
   public PCollection<BeamSQLRow> buildIOReader(Pipeline pipeline) {
-    return PBegin.in(pipeline).apply("decodeRecord", TextIO.Read.from(filePattern))
+    return PBegin.in(pipeline).apply("decodeRecord", TextIO.read().from(filePattern))
         .apply("parseCSVLine",
             new BeamTextCSVTableIOReader(beamSqlRecordType, filePattern, csvFormat));
   }
diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
index eade842a7cf5..9b9cbd226b22 100644
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
+++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableIOWriter.java
@@ -55,6 +55,6 @@ public BeamTextCSVTableIOWriter(BeamSQLRecordType beamSqlRecordType, String file
         BeamSQLRow row = ctx.element();
         ctx.output(beamSQLRow2CsvLine(row, csvFormat));
       }
-    })).apply(TextIO.Write.to(filePattern));
+    })).apply(TextIO.write().to(filePattern));
   }
 }
diff --git a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
index f174b9cae67e..d0397cf7165d 100644
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
+++ b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
@@ -135,8 +135,8 @@ private void setupEnvironment() {
    */
   private void regiesterCoder() {
     CoderRegistry cr = p.getCoderRegistry();
-    cr.registerCoder(BeamSQLRow.class, BeamSqlRowCoder.of());
-    cr.registerCoder(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
+    cr.registerCoderForClass(BeamSQLRow.class, BeamSqlRowCoder.of());
+    cr.registerCoderForClass(BeamSQLRecordType.class, BeamSQLRecordTypeCoder.of());
   }
 
   /**